{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "#将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>8.0</td>\n",
       "      <td>183.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>137.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin  \\\n",
       "0           0          6.0    148.0           72.0           35.0    125.0   \n",
       "1           1          1.0     85.0           66.0           29.0    125.0   \n",
       "2           2          8.0    183.0           64.0           29.0    125.0   \n",
       "3           3          1.0     89.0           66.0           23.0     94.0   \n",
       "4           4          0.0    137.0           40.0           35.0    168.0   \n",
       "\n",
       "    BMI  DiabetesPedigreeFunction   Age  Outcome  \n",
       "0  33.6                     0.627  50.0      1.0  \n",
       "1  26.6                     0.351  31.0      0.0  \n",
       "2  23.3                     0.672  32.0      1.0  \n",
       "3  28.1                     0.167  21.0      0.0  \n",
       "4  43.1                     2.288  33.0      1.0  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = './'\n",
    "data = pd.read_csv(dpath +\"dataFE.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 10 columns):\n",
      "Unnamed: 0                  768 non-null int64\n",
      "Pregnancies                 768 non-null float64\n",
      "Glucose                     768 non-null float64\n",
      "BloodPressure               768 non-null float64\n",
      "SkinThickness               768 non-null float64\n",
      "Insulin                     768 non-null float64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null float64\n",
      "Outcome                     768 non-null float64\n",
      "dtypes: float64(9), int64(1)\n",
      "memory usage: 60.1 KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>383.500000</td>\n",
       "      <td>3.845052</td>\n",
       "      <td>121.656250</td>\n",
       "      <td>72.386719</td>\n",
       "      <td>29.108073</td>\n",
       "      <td>140.671875</td>\n",
       "      <td>32.455208</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>221.846794</td>\n",
       "      <td>3.369578</td>\n",
       "      <td>30.438286</td>\n",
       "      <td>12.096642</td>\n",
       "      <td>8.791221</td>\n",
       "      <td>86.383060</td>\n",
       "      <td>6.875177</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>18.200000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>191.750000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.750000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>121.500000</td>\n",
       "      <td>27.500000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>383.500000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>125.000000</td>\n",
       "      <td>32.300000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>575.250000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>767.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0  Pregnancies     Glucose  BloodPressure  SkinThickness  \\\n",
       "count  768.000000   768.000000  768.000000     768.000000     768.000000   \n",
       "mean   383.500000     3.845052  121.656250      72.386719      29.108073   \n",
       "std    221.846794     3.369578   30.438286      12.096642       8.791221   \n",
       "min      0.000000     0.000000   44.000000      24.000000       7.000000   \n",
       "25%    191.750000     1.000000   99.750000      64.000000      25.000000   \n",
       "50%    383.500000     3.000000  117.000000      72.000000      29.000000   \n",
       "75%    575.250000     6.000000  140.250000      80.000000      32.000000   \n",
       "max    767.000000    17.000000  199.000000     122.000000      99.000000   \n",
       "\n",
       "          Insulin         BMI  DiabetesPedigreeFunction         Age  \\\n",
       "count  768.000000  768.000000                768.000000  768.000000   \n",
       "mean   140.671875   32.455208                  0.471876   33.240885   \n",
       "std     86.383060    6.875177                  0.331329   11.760232   \n",
       "min     14.000000   18.200000                  0.078000   21.000000   \n",
       "25%    121.500000   27.500000                  0.243750   24.000000   \n",
       "50%    125.000000   32.300000                  0.372500   29.000000   \n",
       "75%    127.250000   36.600000                  0.626250   41.000000   \n",
       "max    846.000000   67.100000                  2.420000   81.000000   \n",
       "\n",
       "          Outcome  \n",
       "count  768.000000  \n",
       "mean     0.348958  \n",
       "std      0.476951  \n",
       "min      0.000000  \n",
       "25%      0.000000  \n",
       "50%      0.000000  \n",
       "75%      1.000000  \n",
       "max      1.000000  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAFBlJREFUeJzt3X/QZmV93/H3BxbwF2H5sWEIsFmspK2tirha1MQomEawZS0VNHWEMjvdODUEhzSB2ChWWyMTDQmTit0J1qVjRKQiq9Ig4ZfVFsKiCIgwrhRkt8iuyG8GLPDtH/f14O3mPM9zdtn7B8/zfs3cc59znXPu+3vPLM+Hc65zXSdVhSRJ29pl0gVIkqaTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqdOSSRfwbOy33361YsWKSZchSc8pN9xww4+ratl8+z2nA2LFihVs2LBh0mVI0nNKkrv67OclJklSJwNCktTJgJAkdTIgJEmdDAhJUqeRBkSSO5PcnOTGJBta2z5JLk/y/fa+d2tPknOSbExyU5LDR1mbJGlu4ziDeFNVHVZVK9v6GcAVVXUocEVbBzgaOLS91gDnjqE2SdIsJnGJaRWwri2vA9421H5+DVwLLE1ywATqkyQx+oAo4GtJbkiyprXtX1X3tOUfAfu35QOBu4eO3dTaJEkTMOqR1L9aVZuT/CJweZLbhjdWVSWp7fnAFjRrAJYvX/6sC3zV75//rD9DC88Nf3LipEuQJm6kZxBVtbm9bwEuBl4D3Dtz6ai9b2m7bwYOHjr8oNa27WeuraqVVbVy2bJ5pxKRJO2gkQVEkhcm2XNmGfinwC3AeuCktttJwCVteT1wYrub6QjgwaFLUZKkMRvlJab9gYuTzHzPX1XVXye5HrgwyWrgLuCEtv+lwDHARuAx4OQR1iZJmsfIAqKq7gBe0dF+H3BUR3sB7x1VPZKk7eNIaklSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnUYeEEl2TfLtJF9p64ckuS7JxiSfT7J7a9+jrW9s21eMujZJ0uzGcQZxKvC9ofWzgLOr6iXA/cDq1r4auL+1n932kyRNyEgDIslBwFuBv2zrAY4ELmq7rAPe1pZXtXXa9qPa/pKkCRj1GcSfAX8APN3W9wUeqKon2/om4MC2fCBwN0Db/mDbX5I0AfMGRJIXJtmlLf9KkmOT7NbjuH8GbKmqG3ZCncOfuybJhiQbtm7dujM/WpI0pM8ZxNeB5yU5EPga8G7gMz2Oez1wbJI7gQsYXFr6c2BpkiVtn4OAzW15M3AwQNu+F3Dfth9aVWuramVVrVy2bFmPMiRJO6JPQKSqHgOOAz5ZVccD/2i+g6rqD6vqoKpaAbwTuLKq3gVcBby97XYScElbXt/WaduvrKrq/UskSTtVr4BI8lrgXcBXW9uuz+I7TwdOS7KRQR/Dea39PGDf1n4acMaz+A5J0rO0ZP5deB/wh8DFVfXdJC9mcBbQW1VdDVzdlu8AXtOxz+PA8dvzuZKk0Zk3IKrqGuCaJC9o63cAvzvqwiRJk9XnLqbXJrkVuK2tvyLJJ0demSRpovr0QfwZ8Ju0O4qq6jvAG0ZZlCRp8noNlKuqu7dpemoEtUiSpkifTuq7k7wOqDZAbtu5lSRJC1CfM4j3AO9lMBXGZuCwti5JWsD63MX0YwZjICRJi0ifu5jWJVk6tL53kk+PtixJ0qT1ucT08qp6YGalqu4HXjm6kiRJ06BPQOySZO+ZlST70K9zW5L0HNbnD/0ngP+d5AtAGEyk959GWpUkaeL6dFKfn+QG4E2t6biqunW0ZUmSJq3vpaLbGDw/eglAkuVV9cORVSVJmrh5AyLJKcCZwL0MRlAHKODloy1NkjRJfc4gTgX+flX9nae7SZIWrj53Md0NPDjqQiRJ06XPGcQdwNVJvgo8MdNYVX86sqokSRPXJyB+2F67t5ckaRHoc5vrfwBI8oKqemz0JUmSpoFPlJMkdfKJcpKkTj5RTpLUySfKSZI6+UQ5SVKnOc8gkuwKvLuqfKKcJC0yc55BVNVTwL8aUy2SpCnSpw/iG0n+Avg88OhMY1V9a2RVSZImrk9AHNbePzzUVsCRO78cSdK0mK8PYhfg3Kq6cEz1SJKmxHx9EE8DfzCmWiRJU6TPba5/k+TfJTk4yT4zr5FXJkmaqD59EO9o78NjHwp48c4vR5I0LfrM5nrIOAqRJE2XPs+kPrGrvarOn+e45wFfB/Zo33NRVZ2Z5BDgAmBf4AYGA/F+mmQP4HzgVQwmBnxHVd25Hb9FkrQT9emDePXQ69eADwHH9jjuCeDIqnoFg1tl35LkCOAs4OyqeglwP7C67b8auL+1n932kyRNSJ9LTKcMrydZyuAMYL7jCnikre7WXjPjJ2ZGZ69jEDjnAqvaMsBFwF8kSfscSdKY9ZruexuPAr36JZLsmuRGYAtwOfAD4IGqerLtsonBJIC097sB2vYHGVyGkiRNQJ8+iC8z+D9/GATKS4FeA+faXE6HtbOOi4F/sIN1DtezBlgDsHz58mf7cZKkWfS5zfXjQ8tPAndV1abt+ZKqeiDJVcBrgaVJlrSzhIMYTCFOez8Y2JRkCbAX7Sl223zWWmAtwMqVK738JEkj0ucS0w+B66rqmqr6JnBfkhXzHZRkWTtzIMnzgd9g8KChq4C3t91OAi5py+vbOm37lfY/SNLk9AmILwBPD60/1drmcwBwVZKbgOuBy6vqK8DpwGlJNjLoYziv7X8esG9rPw04o99PkCSNQp9LTEuq6qczK23Mwu7zHVRVNwGv7Gi/A3hNR/vjwPE96pEkjUGfM4itSZ4Z95BkFfDj0ZUkSZoGfc4g3gN8tj00CAa3pnaOrpYkLRx9Bsr9ADgiyYva+iPzHCJJWgDmvcSU5KNJllbVI1X1SJK9k/zHcRQnSZqcPn0QR1fVAzMrVXU/cMzoSpIkTYM+AbFrm2kVeGZMwx5z7C9JWgD6dFJ/FrgiyX9t6yczmGRPkrSA9emkPivJd4A3t6aPVNVloy1LkjRpfc4gAL7Nz6br/vboypE044cfftmkS9AUWv7Bm8f2XX3uYjoB+FsG8yOdAFyX5O1zHyVJeq7rcwbx74FXV9UWGEzCB/wNg4f6SJIWqD53Me0yEw7NfT2PkyQ9h/U5g/jrJJcBn2vr7wAuHV1JkqRp0Ocupt9Pchzwq61pbVVdPNqyJEmT1usupqr6IvDFEdciSZoi9iVIkjoZEJKkTrMGRJIr2vtZ4ytHkjQt5uqDOCDJ64Bjk1wAZHhjVX1rpJVJkiZqroD4IPAB4CDgT7fZVsCRoypKkjR5swZEVV0EXJTkA1X1kTHWJEmaAn3GQXwkybHAG1rT1VX1ldGWJUmatD6T9f0xcCpwa3udmuSjoy5MkjRZfQbKvRU4rKqeBkiyjsGU3+8fZWGSpMnqOw5i6dDyXqMoRJI0XfqcQfwx8O0kVzG41fUNwBkjrUqSNHF9Oqk/l+Rq4NWt6fSq+tFIq5IkTVzfyfruAdaPuBZJ0hRxLiZJUicDQpLUac6ASLJrktvGVYwkaXrMGRBV9RRwe5LlY6pHkjQl+lxi2hv4bpIrkqyfec13UJKDk1yV5NYk301yamvfJ8nlSb7f3vdu7UlyTpKNSW5Kcviz+2mSpGejz11MH9jBz34S+L2q+laSPYEbklwO/Gvgiqr6WJIzGIypOB04Gji0vf4JcG57lyRNwLxnEFV1DXAnsFtbvh6Y91kQVXXPzDMjquph4HvAgcAqYF3bbR3wtra8Cji/Bq4FliY5YPt+jiRpZ+kzWd+/AS4C/ktrOhD40vZ8SZIVwCuB64D927gKgB8B+w997t1Dh21qbZKkCejTB/Fe4PXAQwBV9X3gF/t+QZIXAf8deF9VPTS8raqKwcOHekuyJsmGJBu2bt26PYdKkrZDn4B4oqp+OrOSZAk9/6gn2Y1BOHy2qr7Ymu+duXTU3re09s3AwUOHH9Tafk5Vra2qlVW1ctmyZX3KkCTtgD4BcU2S9wPPT/IbwBeAL893UJIA5wHfq6rhR5auB05qyycBlwy1n9juZjoCeHDoUpQkacz63MV0BrAauBn4beBS4C97HPd64N3AzUlubG3vBz4GXJhkNXAXcELbdilwDLAReAw4uedvkCSNQJ/ZXJ9uDwm6jsGlpdtb38F8x32DwfTgXY7q2L8Y9HdIkqbAvAGR5K3Ap4AfMPiDf0iS366q/zHq4iRJk9PnEtMngDdV1UaAJH8P+CpgQEjSAtank/rhmXBo7gAeHlE9kqQpMesZRJLj2uKGJJcCFzLogziewWhqSdICNtclpn8+tHwv8OtteSvw/JFVJEmaCrMGRFV5m6kkLWJ97mI6BDgFWDG8f1UdO7qyJEmT1ucupi8xGBH9ZeDp0ZYjSZoWfQLi8ao6Z+SVSJKmSp+A+PMkZwJfA56YaZx51oMkaWHqExAvYzCn0pH87BJTtXVJ0gLVJyCOB148POW3JGnh6zOS+hZg6agLkSRNlz5nEEuB25Jcz8/3QXibqyQtYH0C4syRVyFJmjp9ngdxzTgKkSRNlz4jqR/mZ8+g3h3YDXi0qn5hlIVJkiarzxnEnjPL7TnTq4AjRlmUJGny+tzF9Iwa+BLwmyOqR5I0JfpcYjpuaHUXYCXw+MgqkiRNhT53MQ0/F+JJ4E4Gl5kkSQtYnz4InwshSYvQXI8c/eAcx1VVfWQE9UiSpsRcZxCPdrS9EFgN7AsYEJK0gM31yNFPzCwn2RM4FTgZuAD4xGzHSZIWhjn7IJLsA5wGvAtYBxxeVfePozBJ0mTN1QfxJ8BxwFrgZVX1yNiqkiRN3FwD5X4P+CXgj4D/m+Sh9no4yUPjKU+SNClz9UFs1yhrSdLCYghIkjoZEJKkTgaEJKmTASFJ6jSygEjy6SRbktwy1LZPksuTfL+9793ak+ScJBuT3JTk8FHVJUnqZ5RnEJ8B3rJN2xnAFVV1KHBFWwc4Gji0vdYA546wLklSDyMLiKr6OvCTbZpXMRiRTXt/21D7+e2BRNcCS5McMKraJEnzG3cfxP5VdU9b/hGwf1s+ELh7aL9Nre3vSLImyYYkG7Zu3Tq6SiVpkZtYJ3VVFVA7cNzaqlpZVSuXLVs2gsokSTD+gLh35tJRe9/S2jcDBw/td1BrkyRNyLgDYj1wUls+CbhkqP3EdjfTEcCDQ5eiJEkT0OeZ1DskyeeANwL7JdkEnAl8DLgwyWrgLuCEtvulwDHARuAxBs+dkCRN0MgCoqp+a5ZNR3XsW8B7R1WLJGn7OZJaktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVKnqQqIJG9JcnuSjUnOmHQ9krSYTU1AJNkV+M/A0cBLgd9K8tLJViVJi9fUBATwGmBjVd1RVT8FLgBWTbgmSVq0pikgDgTuHlrf1NokSROwZNIFbK8ka4A1bfWRJLdPsp4FZj/gx5MuYhrk4ydNugT9PP9tzjgzO+NTfrnPTtMUEJuBg4fWD2ptP6eq1gJrx1XUYpJkQ1WtnHQd0rb8tzkZ03SJ6Xrg0CSHJNkdeCewfsI1SdKiNTVnEFX1ZJLfAS4DdgU+XVXfnXBZkrRoTU1AAFTVpcClk65jEfPSnaaV/zYnIFU16RokSVNomvogJElTxIBYhOab0iTJHkk+37Zfl2TF+KvUYpPk00m2JLlllu1Jck77d3lTksPHXeNiY0AsMj2nNFkN3F9VLwHOBs4ab5VapD4DvGWO7UcDh7bXGuDcMdS0qBkQi0+fKU1WAeva8kXAUUl2yugcaTZV9XXgJ3Pssgo4vwauBZYmOWA81S1OBsTi02dKk2f2qaongQeBfcdSnTQ7p+MZMwNCktTJgFh8+kxp8sw+SZYAewH3jaU6aXa9puPRzmNALD59pjRZD8zMVvd24MpywIwmbz1wYrub6Qjgwaq6Z9JFLWRTNZJaozfblCZJPgxsqKr1wHnAf0uykUGn4TsnV7EWiySfA94I7JdkE3AmsBtAVX2KwSwLxwAbgceAkydT6eLhSGpJUicvMUmSOhkQkqROBoQkqZMBIUnqZEBIkjoZENIskixN8m/H8D1vTPK6UX+PtL0MCGl2S4HeAdEGcO3If1NvBAwITR3HQUizSDIz0+3twFXAy4G9GQze+qOquqQ9K+My4DrgVQwGcr0ZOB14APgO8ERV/U6SZcCngOXtK97HYKqIa4GngK3AKVX1P8fx+6T5GBDSLNof/69U1T9uc1K9oKoeSrIfgz/qhwK/DNwBvK6qrk3yS8D/Ag4HHgauBL7TAuKvgE9W1TeSLAcuq6p/mORDwCNV9fFx/0ZpLk61IfUT4KNJ3gA8zWCa6f3btrva8wlg8LyNa6rqJwBJvgD8Stv2ZuClQ4/W+IUkLxpH8dKOMCCkft4FLANeVVX/L8mdwPPatkd7fsYuwBFV9fhwo89i0rSyk1qa3cPAnm15L2BLC4c3Mbi01OV64NeT7N0uS/3LoW1fA06ZWUlyWMf3SFPDgJBmUVX3Ad9McgtwGLAyyc3AicBtsxyzGfgo8LfAN4E7GTyRD+B322fclORW4D2t/cvAv0hyY5JfG9XvkbaXndTSTpbkRVX1SDuDuJjBlOoXT7ouaXt5BiHtfB9KciNwC/B/gC9NuB5ph3gGIUnq5BmEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSer0/wHQZFjeu5JJ1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target 分布，看看各类样本分布是否均衡\n",
    "sns.countplot(data.Outcome);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "不同类的数量\n",
      "0.0    500\n",
      "1.0    268\n",
      "Name: Outcome, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#对类别型特征，观察其取值范围及直方图\n",
    "print('不同类的数量')\n",
    "print(data['Outcome'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(614, 9)\n",
      "(614,)\n",
      "(154, 9)\n",
      "(154,)\n"
     ]
    }
   ],
   "source": [
    "#将数据分割训练数据与测试数据\n",
    "#由于各类样本不均衡，训练集和测试集中类的比例要保持不变\n",
    "\n",
    "data_class1 = data[data['Outcome'] == 1]\n",
    "data_class0 = data[data['Outcome'] == 0]\n",
    "\n",
    "y_cls1 = data_class1['Outcome'].values\n",
    "X_cls1 = data_class1.drop('Outcome', axis = 1)\n",
    "\n",
    "# 随机采样20%的数据构建测试样本，其余作为训练样本\n",
    "X_train_cls1, X_test_cls1, y_train_cls1, y_test_cls1 = train_test_split(X_cls1, y_cls1, random_state=33, test_size=0.2)\n",
    "\n",
    "y_cls0 = data_class0['Outcome'].values\n",
    "X_cls0 = data_class0.drop('Outcome', axis = 1)\n",
    "X_train_cls0, X_test_cls0, y_train_cls0, y_test_cls0 = train_test_split(X_cls0, y_cls0, random_state=33, test_size=0.2)\n",
    "\n",
    "X_train = X_train_cls0.append(X_train_cls1)\n",
    "y_train = np.concatenate([y_train_cls0,y_train_cls1])\n",
    "X_test = X_test_cls0.append(X_test_cls1)\n",
    "y_test = np.concatenate([y_test_cls0,y_test_cls1])\n",
    "\n",
    "\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "X_test = ss_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic 回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.45462132 0.53063466 0.56498889 0.5049356  0.43343586]\n",
      "cv logloss is: 0.49772326481629936\n"
     ]
    }
   ],
   "source": [
    "# 采用原始数据（未划分训练集和测试集）\n",
    "y = data['Outcome'].values\n",
    "X = data.drop('Outcome', axis = 1)\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()\n",
    "\n",
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print('logloss of each fold is: ',-loss)\n",
    "print('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优\n",
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） 目标函数为：J = sum(logloss(f(xi), yi)) + C* penalty\n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同： 设置候选参数集合 调用GridSearchCV 调用fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mean_fit_time': array([0.00058951, 0.00076718, 0.00056038, 0.00077472, 0.00084577,\n",
      "       0.0008091 , 0.00105329, 0.00082841, 0.00141568, 0.00089278,\n",
      "       0.00099869, 0.00083103, 0.00083842, 0.00102167]), 'std_fit_time': array([6.07277160e-05, 9.90990373e-05, 4.09482365e-05, 6.30701822e-05,\n",
      "       8.85845634e-05, 2.84494811e-05, 2.99360647e-04, 4.27073099e-05,\n",
      "       9.09879512e-04, 7.27661315e-05, 2.12170482e-04, 2.39563102e-05,\n",
      "       2.82907932e-05, 2.45551433e-04]), 'mean_score_time': array([0.00063672, 0.00056386, 0.00052481, 0.00054851, 0.00052128,\n",
      "       0.00055089, 0.00056391, 0.00052519, 0.00060301, 0.00057554,\n",
      "       0.00048175, 0.00047994, 0.00059114, 0.00064545]), 'std_score_time': array([7.67314704e-05, 8.12938133e-05, 1.67636577e-05, 5.42094017e-05,\n",
      "       2.59929179e-05, 9.35457528e-05, 6.02568987e-05, 6.07759590e-05,\n",
      "       1.71557594e-04, 6.57331740e-05, 2.81849082e-05, 4.28763849e-05,\n",
      "       2.09481548e-04, 1.19587432e-04]), 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
      "                   100, 1000, 1000],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
      "                   'l2', 'l1', 'l2', 'l1', 'l2'],\n",
      "             mask=[False, False, False, False, False, False, False, False,\n",
      "                   False, False, False, False, False, False],\n",
      "       fill_value='?',\n",
      "            dtype=object), 'params': [{'C': 0.001, 'penalty': 'l1'}, {'C': 0.001, 'penalty': 'l2'}, {'C': 0.01, 'penalty': 'l1'}, {'C': 0.01, 'penalty': 'l2'}, {'C': 0.1, 'penalty': 'l1'}, {'C': 0.1, 'penalty': 'l2'}, {'C': 1, 'penalty': 'l1'}, {'C': 1, 'penalty': 'l2'}, {'C': 10, 'penalty': 'l1'}, {'C': 10, 'penalty': 'l2'}, {'C': 100, 'penalty': 'l1'}, {'C': 100, 'penalty': 'l2'}, {'C': 1000, 'penalty': 'l1'}, {'C': 1000, 'penalty': 'l2'}], 'split0_test_score': array([-0.69314718, -0.63333908, -0.68998765, -0.50893127, -0.46889646,\n",
      "       -0.45696888, -0.454128  , -0.45462132, -0.45475936, -0.45482561,\n",
      "       -0.45484636, -0.45485388, -0.45485412, -0.45485678]), 'split1_test_score': array([-0.69314718, -0.64753809, -0.67791053, -0.56138381, -0.51753181,\n",
      "       -0.53112167, -0.52795848, -0.53063466, -0.5305559 , -0.53084305,\n",
      "       -0.53083444, -0.53086817, -0.53086714, -0.53087073]), 'split2_test_score': array([-0.69314718, -0.64956908, -0.67561143, -0.56700403, -0.5500409 ,\n",
      "       -0.55304073, -0.56412357, -0.56498889, -0.56722128, -0.56732711,\n",
      "       -0.56756457, -0.5675809 , -0.56760195, -0.56760649]), 'split3_test_score': array([-0.69314718, -0.64064668, -0.67606524, -0.5439538 , -0.50946893,\n",
      "       -0.50791888, -0.5051896 , -0.5049356 , -0.50486208, -0.50481832,\n",
      "       -0.50481373, -0.50480998, -0.50480536, -0.50480918]), 'split4_test_score': array([-0.69314718, -0.63045642, -0.69291875, -0.49832736, -0.44138407,\n",
      "       -0.43928609, -0.43212933, -0.43343586, -0.43292409, -0.43307585,\n",
      "       -0.43302944, -0.43304404, -0.43304159, -0.4330409 ]), 'mean_test_score': array([-0.69314718, -0.64032592, -0.68248175, -0.53598128, -0.49755577,\n",
      "       -0.49776234, -0.49681097, -0.49782797, -0.49817063, -0.49828402,\n",
      "       -0.49832388, -0.49833756, -0.49834021, -0.498343  ]), 'std_test_score': array([0.        , 0.00752923, 0.00740403, 0.02762411, 0.03810097,\n",
      "       0.04322654, 0.04808678, 0.04824055, 0.04896551, 0.04898042,\n",
      "       0.04905485, 0.04905865, 0.04906494, 0.04906652]), 'rank_test_score': array([14, 12, 13, 11,  2,  3,  1,  4,  5,  6,  7,  8,  9, 10],\n",
      "      dtype=int32), 'split0_train_score': array([-0.69314718, -0.64225616, -0.69075026, -0.53690922, -0.49601176,\n",
      "       -0.49072478, -0.48735048, -0.48719319, -0.48713364, -0.487132  ,\n",
      "       -0.48713136, -0.48713134, -0.48713133, -0.48713133]), 'split1_train_score': array([-0.69314718, -0.63462806, -0.6762319 , -0.51964825, -0.47928089,\n",
      "       -0.47262047, -0.46946766, -0.46935612, -0.4693034 , -0.46930227,\n",
      "       -0.46930171, -0.4693017 , -0.46930169, -0.46930169]), 'split2_train_score': array([-0.69314718, -0.63384566, -0.66527627, -0.51693481, -0.47029071,\n",
      "       -0.46517622, -0.461067  , -0.46094881, -0.4608735 , -0.46087228,\n",
      "       -0.46087146, -0.46087144, -0.46087144, -0.46087144]), 'split3_train_score': array([-0.69314718, -0.63790626, -0.67220967, -0.52491422, -0.48432432,\n",
      "       -0.47929486, -0.47628443, -0.47616806, -0.47611772, -0.47611612,\n",
      "       -0.47611558, -0.47611556, -0.47611556, -0.47611556]), 'split4_train_score': array([-0.69314718, -0.6438239 , -0.69298389, -0.54095455, -0.50366854,\n",
      "       -0.4966652 , -0.49365135, -0.49346598, -0.49341435, -0.49341235,\n",
      "       -0.4934118 , -0.49341178, -0.49341177, -0.49341177]), 'mean_train_score': array([-0.69314718, -0.63849201, -0.6794904 , -0.52787221, -0.48671524,\n",
      "       -0.48089631, -0.47756419, -0.47742644, -0.47736852, -0.477367  ,\n",
      "       -0.47736638, -0.47736636, -0.47736636, -0.47736636]), 'std_train_score': array([0.        , 0.00398647, 0.01071939, 0.00947444, 0.01187089,\n",
      "       0.01151896, 0.01178064, 0.01175485, 0.0117603 , 0.01176001,\n",
      "       0.01176007, 0.01176007, 0.01176007, 0.01176007])}\n"
     ]
    }
   ],
   "source": [
    "# view the complete results (list of named tuples)\n",
    "print(grid.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4968109692804043\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zhangwt/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zhangwt/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8ldX9wPHP92aTBDIJYYSwNyEs8YdaEShUUcGBtqLUDltrW3dFsLYuxL06FLGAVYpWRSJUFBAcqOydsGRJyIDsPe49vz/uEwiYwE1yb27G991Xep/nPufc5/sg3G/OOc9zjhhjUEoppRrK5u0AlFJKtQyaUJRSSrmFJhSllFJuoQlFKaWUW2hCUUop5RaaUJRSSrmFJhSllFJuoQlFKaWUW2hCUUop5Ra+3g6gMUVFRZn4+Hhvh6GUUs3K5s2bTxpjos9XrlUllPj4eDZt2uTtMJRSqlkRkSOulNMuL6WUUm6hCUUppZRbaEJRSinlFppQlFJKuYUmFKWUUm6hCUUppZRbeCWhiEiEiKwUkf3Wa/g5yrYVkWMi8rdq760Vkb0iss36ad84kSullKqNt1ooM4DVxphewGprvzaPAV/U8P5Nxpgh1k+mJ4JUSinlOm8llKuBhdb2QmByTYVEZBgQA3zaSHEp1ehueO0bbnjtG2+HoVSDeSuhxBhj0qztdJxJ4wwiYgOeA+6r5TPmW91dfxYRqe1EInKbiGwSkU0nTpxocOBKudth/2c57P+st8NwiwvmX8sF86/1dhhu0VKupTGvw2NTr4jIKqBDDYdmVd8xxhgRMTWU+x3wP2PMsRryxU3GmFQRCQXeB24G3qwpDmPMXGAuwPDhw2s6j1JKKTfwWEIxxoyr7ZiIZIhIrDEmTURigZrGQC4ELhaR3wEhgL+IFBpjZhhjUq1zFIjIImAktSQUpZRSjcNbXV5JwHRrezqw9OwCxpibjDFxxph4nN1ebxpjZoiIr4hEAYiIHzAJ2NU4YSullKqNtxLKHGC8iOwHxln7iMhwEZl3nroBwCcisgPYBqQCr3syWNX03LriVm5dcau3w1BKVeOV6euNMVnA2Bre3wT8qob3FwALrO0iYJhnI1RKKVVX+qS8Ukopt9CEopRSyi1a1YqNquVITsv3dghKqbNoC0UppZRbaEJRSinlFppQlFJKuYUmFKWUUm6hg/JKedlf3k5xbrSA5zT1WpqexrwObaEopZRyC00oSiml3EK7vFSzNGPRYedGM++OUKol0RaKUkopt9CEopRSyi00oSillHILHUNRzVKAKfF2CEqps2hCUaoRGWMwZWU4CgtxFBZiLywkpNKOv4+h4IN/gakq6LC2TVXFaq/m9P6pbVOtaE11a6pTVcRRbbva51W9ZxzVrwAc1Y6fdb5OPmUA5L36cH3+eJqUlnItVddhP3kcn6iOHj2XJhSlXGCMwZSWYi8owFFYhKPodEJwFBZZCaLgrP1C7EXWfn4ejoIC7MXFYHec8dnRVs/zsZnPeOPS3MoffwCOv/hfL0fScC3lWqquo/LwHk0oSjWEMQZTXIy9WhI4lQgKCnEUnZ0UrCRQUK1skfMYdvt5zyf+ftiC/LEF+ODj68DmU46flODjX4atg8Hm68DmD7bQdvhExmCL6szRr76g3GGj7233gwgg1odZ26desY4JYqOWctb7threq15OanjvVDkQsZ3eB7DZzjzfqTirjju3Nz50OwAjnnjNtf9ATdjGWb8Bmv+1VF1Hn/4jPH4uTSiqWbI5DKFFkPHU01aCKKi5ZVBYCA7HeT9PAgOxhYTgExyMLSQEW0gIfnFd8AkOObVvCwnGJygAGyXYHLn4VGZhK8vAVnocW9FRfMrSEZ9qHxraESJ7WD89IcLaDo8H34BTxY580h+AxCtvd/OfUuPLt/sBEDD8Mi9H0nAt5VqqrsPWJtTj59KEopqdkh076JgBfnbIWbzY+UVf7YvfPyoSW3C1JHAqIYSetR+CLdi5L35+p09gr4CcI5B1ALK/g6zvIGsDZB+E1GNUHz8gONqZKLqNg8juVtLoCRHdwD+40f9slPImTSiq2TDGkPPvf5PxzLMIcDwaxn65pX4f5rBD3veQtQX2H3Qmj6zvnAkk5wiYat1bge2cSSLuwmqtje7O7cB2brk2pVoCTSiqWbDn5XF81iwKV60mZMwYdu9dg8Mm565kDBSknZkssqyfnENgLz9d1i/Y2cKITYAB15zZTdUmotoYg1KqNppQVJNXsmMHqXfdTUVmJu1nPEDE9OnsHD/AedAYKDpZLVlU66bKPggVxac/yCfA2bKI6gW9JzgTRmQPZ9II7aBJQ6kG0oSimixjDNkLF5L57HP4tW9P/NtvEZSQAECAv53EvtkwpyuU5Z2uZPOFsK7OZNHtEqtrykocbTtbdysppTxBE4pqkuy5uRyfOYvCzz4jZOxYOs5+Ap92p8crOrUvJqJtOQyaBlG9T99BFRYHPn7n+GSllKdoQlFNTsn27aTefQ8VJ04Q8+AMwm+5BTmrO6pDZAm5BX6ETXrBS1Eqpc6mCUU1GcYYshcsJPO55/CLiXF2cQ0e/MOCuUcJC61gz6G2hDV+mG73yE39ALjcy3G4g15L09OY16EJRTUJZ3RxjRtLxyfO7OI6Q3ISAOlZQfRtxBiVUuemCUV5Xcm2bRy75x4qT5wkZuaDhN988w+6uM6QkkR+oR/FpfrXV6mmRP9FKq8xxpA9fwGZzz/v7OJa9DZBgwadu1J+Gny/nv/FhvFBb/9m3x2hVEuiCUV5hT03l+MPzqRwzRpCx48j9okn8Gnb9vwV9ywDYH2gTmuiVFPjlYQiIhHAO0A8cBiYaozJqaGcHdhp7R41xlxlvd8NWAxEApuBm40x5WfXV01T8datpN57r9XFNZPwm6edu4uruuSlENWHVL9yjBFKK+w4jMHuMDgcYK/atl6rbztfodLhqLGsw2HOeo8a6p913Fj1qh8/9d7p45XniCkv/SIAZi7Zib+PDV+b4Odrw8/Hhp+17WsT/H1t+Nps+PmI85jPmdu+p7Z/eNzXR5yfXfWezYbtfDMNKFVH3mqhzABWG2PmiMgMa/+BGsqVGGOG1PD+U8ALxpjFIvIq8Evgn54LV7mDcTicXVwvvIBfhw6udXFVV3QSjqzDXHQPmfs/JVMCGTZ/3Jnn4Fxfkuc4ZupZ77zH5dT/S9XU8NY7VTPI2yKc84YtO7EKY8S5ppURax2rqinibdVitN4zYl2vVCtX+7Y5a1+wYRPBR5yvIjZr24aP7fQx57bt1HbVvm+1Vx+bDz4iFBaGAPDLD144c+IBOf3ncPoKbGf88VWbDL9aeanhuFQrd3qv+vmq/4Jydp2qY2f+DiOny1nvFxcHAYYZn87D0+qT2sXFWsUlgQAcy8umc7uIepzJdd5KKFcDl1rbC4G11JxQfkCcfxsuA35Wrf5f0YTSpFXm5JD24EwK164ldPx4Yp943LUurur2LGO/rw+PF+0gKygXSjrSK6Kv86ux2pIhUv0LqOoLR8ypbedxU2379BIhzr9eBpGqL3xz6v3TX0iny5qqr/QzPuf0OZxxOA+a6qscVtv+6vvNgOHCzokYY3AYBwbnq7PV5MBuHDgc1qtVxmEc2B125zbO4wZOHXNYZQ1VrwZjHM5z4HBuV/3PGMCBHUOl8+zOKKtWb5RqMyxXLdhYw6oANus/6YaCek7a2ZQ4cyPL03Z4N46GsnqHD2Qdb7EJJcYYk2ZtpwMxtZQLFJFNQCUwxxjzIc5urlxjTKVV5hjQyaPRqgYp3mJ1cZ08ScxDDxF+089c7+Kq+oyKYv656w3+3SmW0OJ0yjKuRMo78eFv7/BQ1I3ngvnXAvDquFe9HMm5VSU7ZzIyp5Je9ffHvX0LAJ/+dMHpetWn+z/1OcY6VvWeo9q2lbOMOb1q8an3Ty8/bAw4Tm2f/rwzVjo+lTCpdtycKl/9c031ZZSBnyc5f8edf+VTrv8ZnXWtLtWpe5U6neVXyx4EYGSXXnU/UR15LKGIyCqgQw2HZlXfMcYYEantz6erMSZVRLoDn4nITiCvlrK1xXEbcBtAXFxcXaqqBnJ2cc0n8/kX8IuNJX7RIoIGDazbZxjDqqOreGr9k2SQy7XB8fxoyMvcui2ZtrFrPRO4qpGI4CM++OBTexnrWFhg83/k1NfP+Tvr4Nh47wbSQL5+FQC08Qs4T0k3nMtTH2yMGVfbMRHJEJFYY0yaiMQCmbV8Rqr1elBE1gKJwPtAmIj4Wq2UzkDqOeKYC8wFGD58eD1+F1D1UZmTQ9qMByn8/HNCf/xjYh9/rM5dXEfzjzJ7w2zWpa6jT2B7nj2ezpBbFvCXTUUglQSEHPVQ9Eqp+vDW1KtJwHRrezqw9OwCIhIuIgHWdhQwGkg2zrbqGuC6c9VX3lO8ZSuHplxD0ddfE/Pnh+j00ot1SiZl9jL+ue2fTFk6hW2Z25gxcgaLK6MYEhiDvUMi/9uVTkDwMWy2yvN/mFKq0XgrocwBxovIfmCctY+IDBeRqlsq+gGbRGQ7zgQyxxiTbB17ALhHRA7gHFN5o1GjVzUyDgdZ8+Zx5OabET8/uv7nP0TcdFOdxku+Sv2KKUun8I/t/2Bs3FiSJidxU7cr8T34GfS7kg2HczhRUEZg6CEPXolSqj68MihvjMkCxtbw/ibgV9b210CN95QaYw4CIz0Zo6qbypwcjj/wAEVffEnoxInEPvYoPqGhLtdPL0rn6Y1Ps/LISuLbxjN3/Fwu7Hih8+DO95yrK/a/mmWbjxPk50NAyPceuhKlVH3pk/KqwYq3bCH17nuwZ2cT8/CfCf/pT11ulVQ4KliUsoi/b/s7DuPgj4l/ZPqA6fj7+J8ulLwUQjpQ2XE4KxauYWy/9mws0+4upZoaTSitzK0rbgVg/sT5Df4s43CQ9cYbnHjxJfw6daLr4v8QNGCAy/W3ZGzhsW8f40DuAX7U+UfMGDmDzqGdzyxUXgT7V0LiTXxzKIesonImDY5l48YGh6+UcjNNKKpeGtLFlV2azfObnmfpd0uJDY7l5TEvMyZuTM2FD6yCyhLodxXLtqQR7O/DpX3agyYUpZocTSiqzoo3byb1nnuxZ2fT4S8PE3bjjS51cTmMg/f2vcdLW16iuLKYXw36Fb8e9Gva+LWpvVJyErSJpKLLhaz491rG948h0K/25yCUUt6jCUW5zHkX1xuceOkl/Dp3Iv6dxQT27+9S3d1Zu3ni2yfYeXInIzuMZNYFs+ge1v3clSpKYd8KGHgNXx3MJa+kgkmDO7rhSpRSnqAJRbmkMjub4w/MoOjLL2l7+U/o8Oij+ISEnLdefnk+f9v6N97Z+w7hAeHMuXgOl3e73LVB+4NroLwQ+l3Nsq1phAb6cnHvKDdcjVLKEzShqPMq3rSJ1Hvvw56TQ4e//oWwG244b0IwxrD80HKe3fgsOWU53NjnRu5IvIO2/nV4Wj45CQLbURY3mk/f/oIJAzoQ4KvdXUo1VZpQVK2Mw0HW6/M48fLL+HfuTJd3FhPYr9956x3MPcjj6x9nY/pGBkUN4h/j/kH/SNe6xk6pLIe9y6HP5XzxXT4FpZVcMTi2nleilGoMmlBUjSqzszn+pwco+uor2l5+OR0efeS8XVzFFcXM3TGXhbsX0savDQ9f+DDX9roWm9RjQobDX0BpnvPurm3HCWvjx0U9tbtLqaZME4r6geKNG51dXLm5dHjkEcKmXn/OLi5jDGu+X8OcDXNIK0pjcs/J3D3sbiICG7D2QnIS+IdQ2vVHrFr0JVcmdMTPx1szBSmlXKEJpZW58ZXdzo2JPzxmHA6y5r7u7OLq0oUur7163i6uYwXHmLNhDp8f+5yeYT1ZOHEhQ2OGNixIeyXsWQ69J7D2u3yKyu16d5dSzYAmFAVAZVaWs4tr3TraXnEFHR55BJ+Q4FrLl9vLWbB7AXN3zMVHfLhv+H38rN/P8LP5NTyYo19D8UnodxUfbU8jMtifUd09u9KcUqrhNKEoijZs4Pi992HPy6PDo48Qdv25u7i+Of4Ns9fP5nD+YX7c9cfcP+J+OgTXtJZaPSUngW8QxV3H8Nnib7h2WCd8tbtLqSZPE0or5uzimsuJl1/BPy6OLq/PJbBv31rLZxZn8szGZ1hxeAVxoXG8Ou5VRnca7d6gHA5I+Qh6jWP1d0WUVGh3l1LNhSaUVqoyK4vj9/+Joq+/pu2kSXT4619r7eKqdFSyeM9i/rbtb1TYK/jdkN/xi4G/IMDHA0uKHtsIhenOhxm3Hyc6NIAR8drdpVRzoAmlFQooqeTQ5CnY8/Pp8NijhF13Xa1dXNsyt/H4t4+zN2cvozuNZtbIWXRp28VzwSUvBR9/CrqOZc276/nZyDh8bK4v0KWU8h5NKK1MaE4ZYdnl2OJj6DLvdQL79KmxXG5pLi9ueZH3979PTJsYXrj0BcbGja3T6ot1Zoyzu6vHZaw+WEJ5pYNJ+jCjUs2GJpRWpHjjRsKzyykK9qXPe+/V2MXlMA6W7F/Ci1tepLC8kFsH3MpvE3577hmB3eX4Vsg7CpfOYNmO48S2C2RoXLjnz6uUcgtNKK2EcTjImPMUlT5CdvvAGpPJnuw9PPbtY+w4sYOh7Yfy0KiH6BXeq/GCTF4KNl/yu47n8/c2Mf3CeGza3aVUs6EJpZXIW5pE6e7d5LYPxJz1JV1YXsjft/2dRXsWERYQxhMXPcGV3a/0bPfW2YyBlCSIv5hPDpZRYTdMStC7u5RqTjShtAKO4mJOvPACgYMHU1x48NT7xhhWHF7BMxuf4WTJSab2mcofEv9Au4B2jR9kxm7IPgj/90eW7Uijc3gQCZ29EIdSqt70abFWIGveG1RmZhIzYwZYrY5DeYf49cpf86cv/kR0m2gWXbGIh0Y95J1kAs7uLrGRG/dj1h04yRWDYxu3haSUajBtobRwFenpZP3rX7S9/Ce0GZpIuY9hdb9KZiRdQ5BPELMumMX1va/Hx+bldUZSkiDu/1hx2E6lw3ClPsyoVLOjCaWFy3z+eXA4aH/vvRRXFPPiuDIy2xmujL+Se4bfQ1RQE5gS/sQ+OLEHfvIMy3akER/ZhgEd67AQl1KqSdAurxasZMcO8pM+IuLWW/Hr1Il5O+eR2c5w65f+zL54dtNIJgApSwHIjvsxX393kkmDO563u6t/bFv6x2rSUaop0RZKC2WMIePJOfhERxH5619zOO8w83fPJzIwkp3XdfN2eGdKXgqdR7L8iOAwMClBH2ZUqjlyqYUiIqNFJNjaniYiz4tIV8+Gphqi4OOPKdm6lfZ33oktuA1PbniSIJ8gOod09nZoZ8o+BOk7of9VLNt+nB7RwfSJCfV2VEqpenC1y+ufQLGIJAD3At8Bb3osKtUgjrIyMp99joB+/Wg3ZQqrjq7i6+Nfc0fiHfj5uGG9EndKSQLgRJcJbDic7VJ3l1KqaXK1y6vSGGNE5Grgb8aYN0Tkl54MTNVf9oKFVBw/Ttzs2ZQ4ynh649P0Ce/DDX1uYNWRVd4O70zJSyF2CMuP+mEMXOlid9f8ifM9HJhSqq5cbaEUiMiDwDRguYjYgCb2q64CqDxxgqzXXiNk7FiCR13A6ztfJ70onVmjZuFra2JDZnnHIHWzs7trRxp9O4TSs712dynVXLmaUG4AyoBfGmPSgc7AMx6LStVb5ksv4aioIOb++ziUd4gFuxdwdY+rSWyf6O3QfijlIwAyOk9g05EcnVlYqWbO1V9ZC4CXjDF2EekN9AX+U9+TikgE8A4QDxwGphpjcmooZwd2WrtHjTFXWe8vAH4E5FnHfm6M2VbfeFqK0pQU8t7/gIjp0/Hr2pUnV/6GIJ8g7h52t7dDq1lyErQfwEfHnDMZX9FKH2bU259VS+FqQvkCuFhEwoFPgY04Wy031fO8M4DVxpg5IjLD2n+ghnIlxpghtXzG/caY9+p5/hbHGEPGnKfwadeOqN/dzsojK/km7RtmXjCTyKBIb4f3QwUZcPQbuHQGH+1IY0DHtnSLqnnFyJZOx4NUS+FqQhFjTLE1EP8PY8zTIrK9Aee9GrjU2l4IrKXmhKJcVPjZZxSvX0/Mnx+iLMiXpzc+Tb+IfkztPdXbodVsz0eAIa3TBLavOM4DE2tfy141H+tvfd/bIbhNS7mWxrwOV8dQREQuxNkiWV7HujWJMcakWdvpQEwt5QJFZJOIfCsik8869oSI7BCRF0TEA4ubNx+mvJyMp5/Gv0cPwm+4gdd2vEZGcQYzL5jp/Tm6apOcBJG9WJrqHITX8ROlmj9XWyh3AQ8CS4wxu0WkO7DmXBVEZBXQoYZDs6rvWLcjm1o+pqsxJtU632cistMY850VSzrgD8zF2bp5tJY4bgNuA4iLiztXyM1W9qJFVBw5Spe5r3Go8Chv7n6TyT0nM6R9bb2FXlaUBYe/govuYtnONBK6hNElohFWhFRKeZRLCcUY8znwuYiEiEiIMeYg8Mfz1BlX2zERyRCRWGNMmojEApm1fEaq9XpQRNYCicB31Vo3ZSIyH7jvHHHMxZl0GD58eG2Jq9mqzMnh5N//QfDFFxN88cXMXvlrgvyCuGvoXd4OrXZ7l4Oxczx2PLtWZvPQFf28HZFSyg1cnXplkIhsBXYDySKyWUQGNOC8ScB0a3s6sLSGc4ZXdWWJSBQwGki29mOtVwEmA7saEEuzdvKVv+EoLibmgT/x6ZFPWZ+2nj8m/rFpDsRXSU6CsK58kOaM8fJB2t2lVEvg6jjIa8A9xpiuxpg4nNOvvN6A884BxovIfmCctY+IDBeReVaZfsAma/B/DTDHGJNsHXtbRHbivKU4Cni8AbE0W2UHDpDzzjuE3zAVe9eOpwbir+99vbdDq11JLhxc63yYcWc6w7qG0zEsyNtRKaXcwNUxlGBjzKkxE2PM2qrJIuvDGJMFjK3h/U3Ar6ztr4FBtdS/rL7nbkkynn4aW5s2RP3hD7y841UyizN5/tLnm+5APMC+FeCo4PsO49jzWQF/ubK/tyNSSrmJqy2UgyLyZxGJt34eAg6et5bymMIvv6Loiy+Juv12jkoO/979b67pdQ0J0QneDu3ckpOgbSfez+iAiHZ3KdWSuJpQfgFEAx9YP9HWe8oLTGUlGU/NwS8ujrCbfsbs9bNp49eGO4fe6e3Qzq2sAA6swvSdxLKdGYyMjyCmbaC3o1JKuYmrd3nlcJ67ulTjyXn3XcoPfEenV15m5fE1rE9fz59H/ZmIwAhvh3Zu+z8FexlHO4zjwBeFTJ880NsRKaXc6JwJRUScjzPXompuLdV47Pn5nHz5FdqMHIntRxfyzIdX0y+iH9f2utbboZ1fchIER/NeZmdscoifDKzpMSWlVHN1vhbKs40ShXLZyX++ij0vj5gHZ/D3Ha+RWZLJC2NeaNoD8QDlxbB/JWbwDXy0M4MLe0QSFdKqJzhQqsU5Z0KxHmhUTUT5kSNkv/UW7a6ZQmqsP28lvcW1va5lcPRgb4d2ft+thooiDseM5fC6Yn7zox7ejkgp5WYujaFYz3yc3fWVB2wCHrduA1Yelvnss4ifH9F/vJPfrn+wXgPxXpvZNjkJgsL574l4fG1HmThAu7uUamlcfQ7lY8AOLLL2bwTa4JxPawFwpdsjU2coWr+BgpWriL7rTlYVb2ZD+gb+POrPhAeGezu086ssg30rMP2uImlXJqN7RhEe7O/tqJRSbuZqQhlnjBlabX+niGwxxgwVkWmeCEydZux2MubMwbdjLAE3Xc8zH1/PgMgBzWMgHuDg51CWz8H2Yzn2bQl3ju3l7YiUUh7g6nMoPiIysmpHREYAVaPAlW6PSp0h78MPKUtJof299/LanvmcLDnJQ6MeavoD8VWSl0JAW9492R0/H+HH2t2lVIvkagvlV8C/RCQEECAf+KU1/cqTngpOgb2wiMwXXyRoyBAyLuzFW8tmcW3vaxkY1Uye4bBXwN7lmN4TSdqdxSW9omkX5OftqJRSHuDqg40bgUEi0s7az6t2+F1PBKacsua9jv3ESdq/8gp3bHiSEP8Q7kxs4k/EV3f4KyjJ4buosaRtLOVPE/t4OyKllIe4On19OxF5HlgNrBaR56qSi/KciuPHyZ6/gLaTJrGmbSqbMjZx59A7CQsM83ZorkteCn7BLM7pjb+vjXH9alucUynV3Lk6hvIvoACYav3kA166/7T1yHzueQCC//Abnt30LAMjB3JNz2u8HFUdOOywZxmOXuNJ2p3NmD7RhAZqd5dSLZWrYyg9jDHVbyl6RES2eSIg5VS8dSv5y5cTeftvmZv5AVklWbxy2SvNZyAe4Oi3UHSCA1FjydxSxqTBHb0dkVLKg1xtoZSIyEVVOyIyGijxTEjKOBzO24Sjo8m57lIWpSziut7XNZ+B+CrJS8E3kMU5fQj0szG2X3tvR6SU8iBXWyi3AwutcRMBsoGfeyqo1i5/+f8o3b6DDrOf4O6dLxDqH8ofE5vZZM8OB6R8hKP7ZSxNzmds3xja+Lv6100p1Ry51EIxxmwzxiQAg4FBxphEY8x2z4bWOjlKSsh8/nkC+/dn3WBfNmds5q6hdzWvgXiA1M1QcJz9UWPJKipn0mBdSEuplu5809ffU8v7ABhjnvdATK1a9oIFVKalEf7EX3huy18ZFDWIKb2meDusuktZCjY//pPbj2D/Asb01e4upVq68/VBhDZKFAqAioxMTr4+j9Dx43nD91uySrL422V/wyauDnU1EcZA8lIc3X/Eh3uKGNc/hkC/ZnQzgVKqXs43ff0jjRWIghMvvQQVFRT++hr+s+0uru99PQOiBng7rLpL2w65R9nX+7fk7qrQu7uUaiXq/KuviGzxRCCtXcnu3eQtWUL4zdN4MnW+cyB+aDMbiK+SkgTiw6K8gYQG+HJJ7yhvR6SUagT16UsRt0fRyhljyHxyDj7h4WyaGM+WzC3cPexu2gU0w8kIqrq7ul7Ekr2ljB8QQ4Cvdncp1RrUJ6Esd3sUrVzBypUUb9pE6O9+zdMpf2dw1GAm95zs7bDqJzMFsg6wN2IMBaWVXKndXUq1GnVOKMaYhzwRSGvlKC8n85lnCejVkzd7HCenNIdZo2Y1v4H4KilJgLAofxDtgvwY3VO7u5RqLVydHLJARPLunQKLAAAfNklEQVTP+vleRJaISHdPB9mS5fz7LSq+/57y393Eov3vMLXPVPpH9vd2WPWXnISjyyg+2FfJxAEd8PdtpolRKVVnrv5rfxG4H+gEdAbuw7kc8GKcE0eqeqjMzubkP/9J8CWXMJv/0c6/HX9I/IO3w6q/kwcgczd7I8ZQVG5nUoI+zKhUa+JqQrnKGPOaMabAGJNvjJkLTDDGvAM0g0XNm6YTL7+Mo6SE3T8dztbMrc13IL5KylIA3s5PICLYnwu7R3o5IKVUY3I1oRSLyFQRsVk/U4FS65jxUGwtWum+feS++1+Cp17DnMy3GBw9mKt7Xu3tsBomOQl7x2G8fwAmDuyAr492dynVmrj6L/4m4GYgE8iwtqeJSBDwew/F1mIZY8h86mlsISG8e5GQW5bLrAua8UA8QM5hSNvG3ogxlFTYde4upVohV5cAPghcWcvhr9wXTutQ9MUXFK1bB3f+gn+nvsXU3s18IB4g5SMAFuUPITo0gAu6aXeXUq2Nq3d59RaR1SKyy9ofLCJ6+3A9mIoKMp56Gr+uXZnTaRthAWH8PrEFNPKSk7DHDOa/B325fGAHfGz6/KtSrY2rfSyvAw8CFQDGmB3AjfU9qYhEiMhKEdlvvdY4sC8icSLyqYikiEiyiMRb73cTkfUickBE3hER//rG0thy3nmX8oMHOXTLJWzJ2dH8B+IB8lLh2Ab2RVxKWaWDSQn6MKNSrZGrCaWNMWbDWe9VNuC8M4DVxphewGprvyZvAs8YY/oBI3GO4QA8BbxgjOkJ5AC/bEAsjcael8fJV17Bf+RwHvdZwZDoIVzV4ypvh9Vwe5YB8HZ+Ih3aBjIsTm/8U6o1cjWhnBSRHlh3dInIdUBaA857NbDQ2l4I/GCeERHpD/gaY1YCGGMKjTHF4lyM5TLgvXPVb4pO/uMf2PPz+fjKGHLL85r3E/HVJSdhj+rLu4eDuGJwLDbt7lKqVXL12+wO4DWgr4ikAncBv23AeWOMMVUJKR2IqaFMbyBXRD4Qka0i8oyI+ACRQK4xpqqFdAznA5dNWtmhQ2S/vQiuHMfrJSu5sc+N9I3o6+2wGq4wE45+zd6IMZTbHVyhd3cp1Wq5ush3KjAfWANEAPnAdODR2iqIyCqgQw2HZlXfMcYYEanpWRZf4GIgETgKvINzHfulLsZcFcdtwG0AcXFxdanqVpnPPIstIIDnh2UQRhh3JN7htVjcas8yMA4WFQyhU1gQiV2a2VLFSim3cTWhLAVygS3AcVcqGGPG1XZMRDJEJNYYkyYisZweG6nuGLDNumUZEfkQGIVzqpcwEfG1WimdcSa82uKYC8wFGD58uFcewiz69lsKP/uMzFsm8HXpah4f/Tht/dt6IxT3S07CHtaNxUdC+eVFsaeWh1ZKtT6uJpTOxpiJbjxvEs4WzhzrtaZWx0aciSPaGHMC57jJJqtFswa4DudcYrXVbxKM3U7Gk3Pw6RjLI102kxiVyJU9anukp5kpzoZDX7C3+8+pTEdXZlSqlXN1DOVrERnkxvPOAcaLyH5gnLWPiAwXkXkAxhg7zkkoV4vITpwLe71u1X8AuEdEDuAcU3nDjbG5Ve7771O2dy/rJvcgy+Q3/yfiq9v7PzB2Fhcm0jWyDQM7tZBWl1KqXlxtoVwE/FxEDgFlOL/cjTFmcH1OaozJAsbW8P4m4FfV9lcCPziH1Q02sj7nbkz2wkJOvPQyZnBfXmj7LT/rexN9Ivp4Oyz3SU7C3rYLb38fwW9/pN1dSrV2riaUn3g0ihYq67W52LOy+Ne0KCKCIvndkN95OyT3Kc2Dg2vY2+VG7JlwxSDt7lKqtXN1Lq8jng6kpSk/lkr2woXkjhnCJ212MXv47JYzEA+w7xOwl7O4cAjdo4PpFxvq7YiUUl7WQjrzm57M554Fm/DkkKMMbT+USd0neTsk90peij2kA2+ltmfS4I7a3aWU0oTiCcVbtlDw8Qp2T+jN0aAiZl4ws2V94ZYXwYHV7Au/FIexcaU+zKiUQhOK2xmHg4zZT2KiIpjTM4Wf9v1pyxqIB9i/EipLWFyYSJ+YUHrFaHeXUkoTitvlf/QRpbt2kTQ+hOC2LWwgvkryUuxBkfw7rZMupKWUOkUTihs5iovJfP4FSnp2YlHXVO4dfi+h/i3st/eKUtj/KfvDf4QDm87dpZQ6RROKG2X9az6VGRm8cnEhiTHDWt5APMB3n0F5IYuLhtI/ti3do0O8HZFSqonQhOImFRkZZL3xBt+P6MLW2FJmjZrVsgbiqyQvxR4QxlsZcUxK0NaJUuo0TShucuL5F3BUVvDU8DR+1u9n9A7v7e2Q3K+yHPZ+zIHwi6nEl0n6MKNSqhpXn5RX51Cycxd5S5eybkw0jlgffpfQAgfiAQ59AWV5vFM0lITO7YiLbOPtiJRSTYi2UBrIGEPGk09S2S6Y1xOzuW/4fYT4t9BxhZSlOPxCeOtEd51ZWCn1A9pCaaCCTz6hZMsWFk0KZkDcCC7vdrm3Q/IMeyWkLONA+GjKC/y4XO/uUs1MRUUFx44do7S01NuhNFmBgYF07twZPz+/etXXhNIAjrIyMp95ltzO7VgxsIR3W9oT8dUdWQcl2bwjiQyNC6NTWJC3I1KqTo4dO0ZoaCjx8fEt999pAxhjyMrK4tixY3Tr1q1en6FdXg2Q/eabVKSm8srFhfxswDR6hffydkiek5KEwzeIt7P7aHeXapZKS0uJjIzUZFILESEyMrJBLThtodRT5cmTZL36Gnv7tyW9XzC3J9zu7ZA8x+GAlI/4rt2FlBUF6MOMqtmqazK54bVvAHjnNxd6Ipwmp6HJVlso9XTi5Vewl5bwj4uKWvZAPMD366Ewg/8WJzIiPoKYtoHejkipZikk5PT3xMSJEwkLC2PSpJofgL7jjjsYMmQI/fv3JygoiCFDhjBkyBDee++9Op1zy5YtrFixokFxu0pbKPVQuncvue+9x6oRfnQeMJSfdGvh64+lJOHwCWBRTj8euERbJ0q5w/33309xcTGvvfZajcf//ve/A3D48GEmTZrEtm3b6nWeLVu2sGvXLiZOnFjvWF2lLZQ6MsaQMWcOZUG+vPt/MHNkCx6IBzAGkpM41HYkxdKGiQM1oSjlDmPHjiU0tH5z/e3fv58JEyYwbNgwLrnkEvbt2wfA4sWLGThwIAkJCYwZM4aSkhIeffRR3n777Xq1bupKWyh1VLhmLcXffMuicTamDPsFPcN7ejskz0rdAvnH+G/AtYzqHkl0aIC3I1KqwR75aDfJx/PPWy45zVmmaizlXPp3bMtfrhzQ4NhccdtttzFv3jx69OjBunXr+P3vf8+nn37KI488wtq1a4mJiSE3N5egoCAefvhhdu3axYsvvujxuDSh1IEpLyfj6ac5ER3A1tERfJjwW2+H5HkpSzHiy6K8Acy4TO/uUsrbcnNz+fbbb7n22mtPvVdZWQnA6NGjueWWW7j++uu55pprGj02TSguuHXFrQA8lzmGisOHmXe9jXtG3U+wX7CXI/OwU91dwykqC2XiwA7ejkgpt3C1JdEU7/IyxhAVFVXjmMrrr7/O+vXrWbZsGUOHDmXr1q2NGpsmFBfc+MpubHZDZvZudnf3xW/0CCbGe36Ay+vSd0LOId7zn8jonlFEBPt7OyKlWr3w8HBiY2NZsmQJU6ZMweFwsHPnThISEjh48CCjRo3iggsuYPny5aSmphIaGkpBQUGjxKaD8i5ql1OGvaCQhZfZmNlSp6Y/W0oSRmwszk/QlRmVcrOLL76Y66+/ntWrV9O5c2c++eQTl+suXryYV199lYSEBAYMGMCyZcsAuPvuuxk0aBCDBg1izJgxDBw4kMsuu4zt27eTmJiog/JNgW+5nZC8ClYlCmMunU6PsB7eDqlxJCdxJCSRgvJ2TOiv3V1KNVRhYeGp7S+//NKlOvHx8ezateuM97p3715jAkpKSvrBe9HR0WzatKmOkdaPJhQXhGWVUeoPq3/cnkUJv/F2OI0jcw+c3Mt7vr/m4l7RtGtTv8nilGrOmtLYSXOgCcUF6/v5sCvWcPuYGS1/IL5KivM3nXcLhzBDu7uUUi7QhOKCr/pDqZ8vE7pO8HYojSc5iaPBg8mtjGR8/xhvR6OUagY0objglq/9KfVr+MRpzUbWd5Cxk/d9fs6lvaMJDdTuLqXU+WlCccE7fxgIwGQvx9ForO6u94oSeSBBH2ZUSrlGE4r6oeQkjgX1Jcsew9i+7b0djVLeM/8K5+uty70bRzPhledQRCRCRFaKyH7rNbyWcnEi8qmIpIhIsojEW+8vEJFDIrLN+hnSmPG3aLlH4fgWlpQN47K+7QkO0N85lHKXxp6+fsmSJTzzzDMNjttV3vq2mAGsNsbMEZEZ1v4DNZR7E3jCGLNSREIAR7Vj9xtjPPuUTmuU8hEA75UM4wFdmVEpj3HX9PWVlZX4+tb8VT5lyhT3BOsibyWUq4FLre2FwFrOSigi0h/wNcasBDDGFOIl8yfO99apG19yEmmBPTjh6MSYPtrdpZSnjB07lrVr19ar7kUXXcSIESP48ssvmTZtGt26dWP27NmUl5cTHR3NW2+9Rfv27Zk3b96pmYanTZtGZGQkGzduJD09neeee87tCcdbCSXGGJNmbacDNd2X2hvIFZEPgG7AKmCGMcZuHX9CRB4GVlvvl3k66BYvPw3z/XqWyFTG9YshyN/H2xEp5Rkfz3DOVXc+6Tucr1VjKefSYRD8ZE7D4qoDu91+6gn4nJwcrrrqKkSEV199leeee46nnnrqB3UyMzNZt24dO3fuZOrUqc0noYjIKqCm+TpmVd8xxhgRMTWU8wUuBhKBo8A7wM+BN4AHcSYif2AuztbNo7XEcRtwG0BcXFw9rqQV2bMMwfBB6TD+pA8zKtWk3XDDDae2jx49ytSpU0lPT6esrIzevXvXWGfy5MmICIMHDyY1NdXtMXksoRhjxtV2TEQyRCTWGJMmIrFAZg3FjgHbjDEHrTofAqOAN6q1bspEZD5w3znimIsz6TB8+PCaEpeqkryUDP84Mkw8l/SO9nY0SnmOqy2JJnyXV3Dw6Vk77rjjDmbOnMnll1/OqlWrmDOn5usLCDi9QJ4x7v869NZsw0nAdGt7OrC0hjIbgTARqfpmuwxIBrCSEOJ80nAysKuG+qouik5ijqzjw/LhjO8fQ6Cfdncp1Vzk5eXRqVMnjDEsXLjQa3F4K6HMAcaLyH5gnLWPiAwXkXkA1ljJfcBqEdkJCPC6Vf9t672dQBTweCPH3/LsWY4YB0vLhjMpQbu7lPK0hkxff7a//vWvTJkyhREjRhAT472pksQTzZ6mavjw4aaxpnFudv59DSePpjC24kU2PjQef19dKke1LCkpKfTr169ulZpwl5en1PTnJCKbjTHDz1dXn1pTUJKDOfQ5SfbLmTCogyYTpaq0okTiDvrNoWDvCsRRydLy4UzShxmVUvWkLRQFyUvJ9m3PUenLhT0ivR2NUqqZ0hZKa1dWgPnuM5ZVDGPioI74+ehfCaVU/ei3R2u37xPEXkZS+Qiu1IcZlVINoAmltUtJIs8ngiNtBnFBd+3uUqq6W1fcyq0rbvV2GM2GJpTWrLwYs38lyyuG8ZPBHfGxtZIVKZXykqrp67dt28aFF17IgAEDGDx4MO+8884Pyrpj+nqALVu2sGLFCrfEfz46KN+aHViFVBSzrHIEd+ndXUo1mjZt2vDmm2/Sq1cvjh8/zrBhw5gwYQJhYWGnyrg6ff35bNmyhV27djFx4kS3xH4u2kJpzVKSKLC15XDwEIZ3rXGNM6WUB/Tu3ZtevXoB0LFjR9q3b8+JEydcrr9//34mTJjAsGHDuOSSS9i3bx8AixcvZuDAgSQkJDBmzBhKSkp49NFHefvtt+vVuqkrbaG0VpVlmL0fs6JyBBMSO2PT7i7Vijy14Sn2ZO85b7mqMq6Mo/SN6MsDI2taJ/DcNmzYQHl5OT169HC5zm233ca8efPo0aMH69at4/e//z2ffvopjzzyCGvXriUmJobc3FyCgoJ4+OGHT62J4mmaUFqbqqkk/u8PSHkhyytH8Aft7lLKK9LS0rj55ptZuHAhNptrHUa5ubl8++23XHvttafeq6ysBGD06NHccsstXH/99VxzzTUeiflcNKG0VilJFEswh0KHMzQu7PzllWpBXG1JVLVMPLFqa35+PldccQVPPPEEo0aNcrmeMYaoqKgax1Ref/111q9fz7Jlyxg6dChbt251Z8jnpWMorZFx4NiznE8qhzIhIQ7nKgBKqcZSXl7OlClTuOWWW7juuuvqVDc8PJzY2FiWLFkCgMPhYPv27QAcPHiQUaNG8dhjjxEeHk5qaiqhoaEUFBS4/RpqogmlNSrNw1aay//sI5ikDzMq1ejeffddvvjiCxYsWHDqduC63MW1ePFiXn31VRISEhgwYADLli0D4O6772bQoEEMGjSIMWPGMHDgQC677DK2b99OYmKiDsorDyjOokSCONTuAgZ1auftaJRqNQoLCwGYNm0a06ZNc6lOfHw8u3aduYZg9+7da1w/JSkp6QfvRUdH01jLdmhCaWV2H8+ld8VJVttHMD4hXru7lDoHT4ydtGTa5dXKtDFF+FGp3V1KKbfThNLKtHXkU2Z8ORz+f/SPbevtcJRSLYh2ebV09krITIbUTXBsE+0cuax2JDIuobt2dyml3EoTSktTkA7HNsIxZwLh+BaoKHYeaxPFYWJ5uvIG/pmgDzMqpdxLE0pzVlEKadutBLIRUjdD3vfOYzY/iB0MQ2/BdBpGdngCe0sj+PP8JCrFh94xod6NXalm4MjNtwDQ9d9vejmS5kETSnNhDOQcsloeVgJJ3wWOCufxdnHQeQQlw27jUEB/tlbGsfdkOXuPFrB/UyHZRQesD+rA7wM/Bn7upQtRqvUKCQmhsLCQbdu2cfvtt5Ofn4+Pjw+zZs3ihhtuOKPsHXfcwbp16ygvL+fQoUP06dMHgIceesjlhyGXLFnCgQMHuP/++91+LTXRhNJUleZB6pbTCSR1ExRnOY/5BVMZO4STA3/Ffr++bKjowbYcf/buKyBzcxlQBuwnJMCXXjEh/Lh/DL1iQukTE4p55yYibEXevDKlWj13Tl9fWVmJr2/NX+VTpkxxf/DnoAmlKXDYITPFGji3xj9O7AUMAKVhvUiPvISU9n34uqwbn2dHcnRf+anqgX6F9GofykW9ougTE0rvmFB6dwilY7vAHwy879ZkopTX9e7d+9R29enrqyeUc7nooosYMWIEX375JdOmTaNbt27Mnj2b8vJyoqOjeeutt2jfvj3z5s07NdPwtGnTiIyMZOPGjaSnp/Pcc8+5PeFoQvGGgowzk8fxrVDufIK2zD+MY20GsDN8Ol+WxLMyrxP56cGQDv4+NrpHBzMkPpQbOoTSq30IfTqE0jm8ja62qFQdpM+eTVnK+aevL93jLFM1lnIuAf360mHmzDrHUp/p6wHsdvupJ+BzcnK46qqrEBFeffVVnnvuOZ566qkf1MnMzGTdunXs3LmTqVOnakJpdirLIG3HqXEPk7oJyT0KgF18OObfk63mR3xR2ZXN9p4cKY3Bp9BGt6hgencJ4RfDrRZHTCjxkW3w9dFHh5RqKeozfX2V6mMuR48eZerUqaSnp1NWVnZGC6i6yZMnIyIMHjyY1NTUBsVeE00o7mQM5B6BY5sw32+g/MhG/E7sxGYNnGdKFJvsPdlsv5itjl4kE09MmzArYYRwT0wofTqE0i0qmABfHy9fjFItl6stCU/e5VXf6eurBAcHn9q+4447mDlzJpdffjmrVq1izpw5NdYJCAg4tW2MqXvQ56EJpSHKCjCpm8k/8C0Vh9cTfGIrQRU5AJSYAHaabmx1TGSroyfpoQOJ6BBH75hQ+seEMjkmlJ7tQwjyb9zEMSBWJ4NUytsaMn19TfLy8ujUqRPGGBYuXOiGCOtHE4qLjL2S7CM7yd77DY7vN9Auazvtyw5hw9AOOODoyBrHIL7z70tBdCKBnQbSOzaMkTGh3NQ+hNBAP29fgtOty70dgVKtXtX09VlZWSxYsADg1FT29fHXv/6VKVOmEBERwaWXXkpaWpobo3WdeKLZ01QNHz7c1Gca5/WvTGdA1ieEUAJArglml/TmeOhAStsnEhg/kvjOnegdE0JYG393h62UcoOUlBT69etXpzqt8cHGmv6cRGSzMWb4+epqC8UFjrad2MlEKjoMI6THKLr0HMTo0ACdC0upFq41JRJ30ITiggunz/Z2CEop1eR55R5UEYkQkZUist96Da+hzBgR2Vbtp1REJlvHuonIehE5ICLviIj2MymllJd566GGGcBqY0wvYLW1fwZjzBpjzBBjzBDgMqAY+NQ6/BTwgjGmJ5AD/LJxwlZKNWetacy4Phr65+OthHI1UHVv20Jg8nnKXwd8bIwpFufAxWXAe3Wor5Rq5QIDA8nKytKkUgtjDFlZWQQGBtb7M7w1hhJjjKm6ry0diDlP+RuB563tSCDXGFNp7R8DOtVWUURuA24DiIuLq3fASqnmrXPnzhw7dowTJ054O5QmKzAwkM6dO9e7vscSioisAjrUcGhW9R1jjBGRWn9lEJFYYBDwSX3iMMbMBeaC87bh+nyGUqr58/Pzo1u3bt4Oo0XzWEIxxoyr7ZiIZIhIrDEmzUoYmef4qKnAEmOMtfAHWUCYiPharZTOgPsnpVFKKVUn3hpDSQKmW9vTgaXnKPtT4D9VO8bZAboG57iKK/WVUko1Am8llDnAeBHZD4yz9hGR4SIyr6qQiMQDXYDPz6r/AHCPiBzAOabyRiPErJRS6hxa1dQrInICOFLP6lHASTeG400t5VpaynWAXktT1VKupaHX0dUYE32+Qq0qoTSEiGxyZS6b5qClXEtLuQ7Qa2mqWsq1NNZ16GpNSiml3EITilJKKbfQhOK6ud4OwI1ayrW0lOsAvZamqqVcS6Nch46hKKWUcgttoSillHILTSh1ICKPicgOazr9T0Wko7djqi8ReUZE9ljXs0REwrwdU32IyPUisltEHCLSLO/GEZGJIrLXWo7hBzNvNxci8i8RyRSRXd6OpSFEpIuIrBGRZOvv1p3ejqm+RCRQRDaIyHbrWh7x6Pm0y8t1ItLWGJNvbf8R6G+M+a2Xw6oXEfkx8JkxplJEngIwxjzg5bDqTET6AQ7gNeA+Y0zd13j2IhHxAfYB43FOdLoR+KkxJtmrgdWDiFwCFAJvGmMGejue+rKmg4o1xmwRkVBgMzC5mf43ESDYGFMoIn7AV8CdxphvPXE+baHUQVUysQQDzTYbG2M+rTZj87c450RrdowxKcaYvd6OowFGAgeMMQeNMeXAYpzLOzQ7xpgvgGxvx9FQxpg0Y8wWa7sASOEcM5o3Zcap0Nr1s3489r2lCaWOROQJEfkeuAl42NvxuMkvgI+9HUQr1Qn4vtr+OZdjUI3Lmv4pEVjv3UjqT0R8RGQbzkl4VxpjPHYtmlDOIiKrRGRXDT9XAxhjZhljugBvA7/3brTndr5rscrMAipxXk+T5Mp1KOVuIhICvA/cdVbvRLNijLFbK992BkaKiMe6I721wFaTda5p98/yNvA/4C8eDKdBznctIvJzYBIw1jThwbQ6/DdpjlJxToBaRZdjaAKs8Yb3gbeNMR94Ox53MMbkisgaYCLgkRsntIVSByLSq9ru1cAeb8XSUCIyEfgTcJUxptjb8bRiG4FeItJNRPxxrk6a5OWYWjVrIPsNIMUY8/z5yjdlIhJddQeniAThvPnDY99bepdXHYjI+0AfnHcVHQF+a4xplr9NWlP/B+BcsAzg2+Z4x5qITAFeAaKBXGCbMWaCd6OqGxG5HHgR8AH+ZYx5wssh1YuI/Ae4FOfMthnAX4wxzW5pCRG5CPgS2Inz3zrATGPM/7wXVf2IyGBgIc6/WzbgXWPMox47nyYUpZRS7qBdXkoppdxCE4pSSim30ISilFLKLTShKKWUcgtNKEoppdxCE4pSbiQihecvdc7674lId2s7REReE5HvRGSziKwVkQtExF9EvhARfTBZNSmaUJRqIkRkAOBjjDlovTUP52SLvYwxw4BbgShrEsnVwA3eiVSpmmlCUcoDxOkZa86xnSJyg/W+TUT+Ya1Fs1JE/ici11nVbgKWWuV6ABcADxljHADGmEPGmOVW2Q+t8ko1GdpkVsozrgGGAAk4nxzfKCJfAKOBeKA/0B7n1Oj/suqMBv5jbQ/A+dS/vZbP3wWM8EjkStWTtlCU8oyLgP9YM71mAJ/jTAAXAf81xjiMMenAmmp1YoETrny4lWjKrQWglGoSNKEo1XSUAIHW9m4gwVrRsTYBQKnHo1LKRZpQlPKML4EbrMWNooFLgA3AOuBaaywlBudkilVSgJ4AxpjvgE3AI9bst4hIvIhcYW1HAieNMRWNdUFKnY8mFKU8YwmwA9gOfAb8yerieh/nqozJwFvAFiDPqrOcMxPMr4AY4ICI7AIW4Fx1D2CMVV6pJkNnG1aqkYlIiDGm0GplbABGG2PSrfUq1lj7tQ3GV33GB8AMY8y+RghZKZfoXV5KNb5l1qJH/sBjVssFY0yJiPwF55ryR2urbC3E9aEmE9XUaAtFKaWUW+gYilJKKbfQhKKUUsotNKEopZRyC00oSiml3EITilJKKbfQhKKUUsot/h8dstI+SxcHdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#pd.DataFrame(grid.cv_results_).to_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#cvresult = pd.DataFrame.from_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#test_means = cv_results['mean_test_score']\n",
    "#test_stds = cv_results['std_test_score'] \n",
    "#train_means = cvresult['mean_train_score']\n",
    "#train_stds = cvresult['std_train_score'] \n",
    "\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'neg-logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用LogisticRegressionCV实现正则化的 Logistic Regression\n",
    "## L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.1, 1, 10, 100, 1000], class_weight=None,\n",
       "           cv=5, dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [0.01,0.1,1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train, y_train)    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1.0: array([[-0.68998765, -0.4688953 , -0.45412616, -0.45475805, -0.45484648,\n",
       "         -0.45485331],\n",
       "        [-0.67791053, -0.51753465, -0.5279537 , -0.5305571 , -0.53083937,\n",
       "         -0.53086826],\n",
       "        [-0.67561143, -0.5500402 , -0.56412064, -0.56722718, -0.56756529,\n",
       "         -0.56760168],\n",
       "        [-0.67606524, -0.50946828, -0.50518911, -0.50485704, -0.50481734,\n",
       "         -0.5048106 ],\n",
       "        [-0.69291875, -0.44138123, -0.4321306 , -0.43292818, -0.43303044,\n",
       "         -0.43303513]])}"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XuYXXV97/H3Z2YyCbmRhEx2YhJJgASyEYhlCFooIrDTeM4pcNRyUatYlXNOy0N97LHCsac+Re3p7ahtpa2I11ZFSovGqieEu4rBTGwgZELCMIAkQGZygSTkOjPf88deA5thMrMze9asmb0/r+dZsvdv/dba36XtfFi3308RgZmZ2VDVZV2AmZmNbQ4SMzOriIPEzMwq4iAxM7OKOEjMzKwiDhIzM6uIg8TMzCriIDEzs4o4SMzMrCINWRcwEmbOnBkLFizIugwzszFl3bp1OyKiabB+NREkCxYsoKWlJesyzMzGFEnPlNPPl7bMzKwiDhIzM6uIg8TMzCriIDEzs4o4SMzMrCIOEjMzq4iDxMzMKuIgGcDKR57jn9eU9Ri1mVnNcpAM4P899jx/c88T9PR4Xnszs6NxkAxgeX42nXsPsX7ri1mXYmY2ajlIBvD2U2dRXydWt27PuhQzs1HLQTKA4yeO49yFMxwkZmYDcJAMopDP0daxj6d2vJx1KWZmo1KqQSJphaTNktok3XCUPldIapW0UdK3k7a3S1pfshyUdHmy7uuSnipZtzTNYyjkcwCsbn0hzZ8xMxuzUhtGXlI9cDNQALYCayWtjIjWkj6LgBuB8yJit6RZABFxH7A06TMDaAPuKtn9xyPijrRqLzVv+kSWzJnK6tbtXHvBySPxk2ZmY0qaZyTLgLaIaI+Iw8BtwGV9+nwEuDkidgNEREc/+3k38OOI2J9irQNans/R8sxuduw7lFUJZmajVppBMhd4tuT71qSt1GJgsaSfSVojaUU/+7kK+E6fts9KelTS5yWN7+/HJV0rqUVSS2dn51CPAShe3oqAezf1l3NmZrUt65vtDcAi4ELgauDLkqb1rpQ0BzgDWFWyzY3AacA5wAzgE/3tOCJuiYjmiGhuahp0psgBnf6Gqcyddhx3+ektM7PXSTNItgHzS77PS9pKbQVWRsSRiHgK2EIxWHpdAdwZEUd6GyLi+Sg6BHyN4iW0VEnikiWz+GlbJwcOd6f9c2ZmY0qaQbIWWCRpoaRGipeoVvbp8z2KZyNImknxUld7yfqr6XNZKzlLQZKAy4HH0ii+r0J+NgeP9PCTJyq7TGZmVm1SC5KI6AKuo3hZahNwe0RslHSTpEuTbquAnZJagfsoPo21E0DSAopnNA/02fW3JG0ANgAzgc+kdQylzj1pBlMmNPjlRDOzPlJ7/BcgIn4E/KhP25+UfA7gY8nSd9unef3NeSLiomEvtAzj6uu46LRZ3PN4B909QX2dsijDzGzUyfpm+5hSyOfY9fJh1j2zO+tSzMxGDQfJMXjb4ibG1ctvuZuZlXCQHIMpE8bx1pNnsrp1O8WrcmZm5iA5RoV8jqd37qetY1/WpZiZjQoOkmNUWFIcxNEvJ5qZFTlIjtHs4ydw5rzjHSRmZgkHyRAsz+d45NkX2b7nYNalmJllzkEyBIX8bADu3uSzEjMzB8kQLM5N5o0zJvotdzMzHCRDIolCPsdDbTvZd6gr63LMzDLlIBmiQj7H4e4eHtziQRzNrLY5SIao+cTpTJs4jrs2+i13M6ttDpIhaqiv4+LTctz7eAdHunuyLsfMLDMOkgoU8jn2HOxi7VO7si7FzCwzDpIKXLB4JuMb6vxyopnVNAdJBSY2NnD+KR7E0cxqm4OkQoV8jm0vHmDT83uzLsXMLBOpBomkFZI2S2qTdMNR+lwhqVXSRknfLmnvlrQ+WVaWtC+U9HCyz+8m88Fn5uIlOST8cqKZ1azUgkRSPXAz8A4gD1wtKd+nzyLgRuC8iDgd+GjJ6gMRsTRZLi1p/wvg8xFxCrAb+FBax1COpinj+bU3TucuT3ZlZjUqzTOSZUBbRLRHxGHgNuCyPn0+AtwcEbsBIqJjoB1KEnARcEfS9A3g8mGteggK+Rwbn9vDthcPZF2KmdmISzNI5gLPlnzfmrSVWgwslvQzSWskrShZN0FSS9LeGxYnAC9GRO+4JP3tc8QV8sU5Su725S0zq0FZ32xvABYBFwJXA1+WNC1Zd2JENAPvAb4g6eRj2bGka5MgaunsTHcYk5ObJnNS0yTfJzGzmpRmkGwD5pd8n5e0ldoKrIyIIxHxFLCFYrAQEduSf7YD9wNvBnYC0yQ1DLBPku1uiYjmiGhuamoaniMaQCGfY037Tl46cCT13zIzG03SDJK1wKLkKatG4CpgZZ8+36N4NoKkmRQvdbVLmi5pfEn7eUBrFF/WuA94d7L9B4Dvp3gMZVuez9HVE9y/ecDbPGZmVSe1IEnuY1wHrAI2AbdHxEZJN0nqfQprFbBTUivFgPh4ROwElgAtkh5J2v88IlqTbT4BfExSG8V7Jl9J6xiOxdL505k5udFvuZtZzWkYvMvQRcSPgB/1afuTks8BfCxZSvs8BJxxlH22U3wibFSprxOXLMnx748+z6GubsY31GddkpnZiMj6ZntVKeRz7DvUxZp2D+JoZrXDQTKMzjtlJseNq2e1X040sxriIBlGE8bVc8Himdzd2uFBHM2sZjhIhlkhP5sX9hxkw7aXsi7FzGxEOEiG2UWnzaJOcNdGP71lZrXBQTLMZkxq5JwFM/yWu5nVDAdJCgr5HJu37+VXO/dnXYqZWeocJClYnp8N4KHlzawmOEhS8MYTJnJqboovb5lZTXCQpKSQz7H26V3sfvlw1qWYmaXKQZKSQj5HT8A9j3sQRzOrbg6SlJwx93hmT53gt9zNrOo5SFJSVycuyc/iwS07OHikO+tyzMxS4yBJUSE/mwNHuvlZ246sSzEzS42DJEVvOWkGk8c3+OktM6tqDpIUjW+o522nNnH3pg56ejyIo5lVJwdJypbnc+zYd4j/ePbFrEsxM0uFgyRlF546i4Y6+S13M6taqQaJpBWSNktqk3TDUfpcIalV0kZJ307alkr6edL2qKQrS/p/XdJTktYny9I0j6FSxx83jrecdILvk5hZ1UotSCTVAzcD7wDywNWS8n36LAJuBM6LiNOBjyar9gPvT9pWAF+QNK1k049HxNJkWZ/WMQyXQj5He+fLPNm5L+tSzMyGXZpnJMuAtohoj4jDwG3AZX36fAS4OSJ2A0RER/LPLRHxRPL5OaADaEqx1lRdks8B+KzEzKpSmkEyF3i25PvWpK3UYmCxpJ9JWiNpRd+dSFoGNAJPljR/Nrnk9XlJ4/v7cUnXSmqR1NLZ2VnZkVRo7rTjOP0NUx0kZlaVsr7Z3gAsAi4Erga+XHoJS9Ic4J+AD0ZET9J8I3AacA4wA/hEfzuOiFsiojkimpuasj+ZKeRz/PJXu+nceyjrUszMhlWaQbINmF/yfV7SVmorsDIijkTEU8AWisGCpKnAD4FPRsSa3g0i4vkoOgR8jeIltFGvkM8RAfds8lmJmVWXNINkLbBI0kJJjcBVwMo+fb5H8WwESTMpXupqT/rfCXwzIu4o3SA5S0GSgMuBx1I8hmGTnzOVudOO8+UtM6s6qQVJRHQB1wGrgE3A7RGxUdJNki5Nuq0CdkpqBe6j+DTWTuAK4ALgmn4e8/2WpA3ABmAm8Jm0jmE4SaKQz/HTth3sP9yVdTlmZsNGEdU/dEdzc3O0tLRkXQYPte3gPbc+zD++72xWvGl21uWYmQ1I0rqIaB6sX9Y322vKOQtnMHWCB3E0s+riIBlB4+rruOi0Wdz7+Ha6unsG38DMbAxwkIywQn42u/cfoeWZ3VmXYmY2LBwkI+xtpzbRWF/ny1tmVjUcJCNs8vgGfv2U4iCOtfCgg5lVPwdJBgr5HL/atZ8t2z2Io5mNfWUFiaTzJE1KPr9P0ucknZhuadXrkiW9gzh6jhIzG/vKPSP5B2C/pLOAP6Q4gOI3U6uqyuWmTuCs+dN8n8TMqkK5QdIVxQv6lwFfjIibgSnplVX9ludzPLL1JV546WDWpZiZVaTcINkr6UbgfcAPJdUB49Irq/oVeuco8SCOZjbGlRskVwKHgA9FxAsUR/L9q9SqqgGLZk1mwQkTfXnLzMa8ss9IgL+JiJ9IWgwsBb6TXlnVr3cQx58/uYO9B49kXY6Z2ZCVGyQPAuMlzQXuAn4H+HpaRdWKQn42R7qDB7ZkO4OjmVklyg0SRcR+4J3A30fEbwNvSq+s2nD2idOZManRl7fMbEwrO0gkvRV4L8VZC49lWzuK+jolgzh2cMSDOJrZGFVuGHyU4lzpdyaTU51EcSIqq1Ahn2PvwS4ebt+VdSlmZkNSVpBExAMRcSlws6TJEdEeEdenXFtNuGBRExPG1fktdzMbs8odIuUMSf8BbARaJa2TdHoZ262QtFlSm6QbjtLnCkmtkjZK+nZJ+wckPZEsHyhpP1vShmSff5vM3T5mHddYz/mnNHkQRzMbs8q9tPUl4GMRcWJEvJHiMClfHmgDSfXAzcA7gDxwtaR8nz6LKF4yOy8iTqd4CQ1JM4BPAecCy4BPSZqebPYPwEeARcmyosxjGLWW53M899JBNj63J+tSzMyOWblBMikiXrknEhH3A5MG2WYZ0JZcBjsM3EZxiJVSHwFujojdyX47kvbfBFZHxK5k3WpghaQ5wNSIWJMM2fJN4PIyj2HUumjJLCT89JaZjUnlBkm7pP8taUGy/DHQPsg2c4FnS75vTdpKLQYWS/qZpDWSVgyy7dzk80D7HHNmTh7P2W+c7iAxszGp3CD5XaAJ+LdkaUraKtVA8fLUhcDVwJclTRuG/SLpWkktklo6O0f/C3+FfI7W5/fw7K79WZdiZnZMyn1qa3dEXB8Rv5Ysf9B7OWoA24D5Jd/nJW2ltgIrI+JIRDwFbKEYLEfbdlvyeaB99tZ8S0Q0R0RzU1PTYIeYueWnzwbgbg/iaGZjTMNAKyX9ADjqo0TJI8FHsxZYJGkhxT/2VwHv6dPnexTPRL4maSbFS13tFOc7+bOSG+zLgRsjYpekPZLeAjwMvB/4u4GOYaxYOHMSp8yazOrW7XzwvIVZl2NmVrYBgwT466HuOCK6JF0HrALqga8mLzPeBLRExMpk3XJJrUA38PGI2Akg6dMUwwjgpojofWPv9yiO83Uc8ONkqQqFfI5bHmznpf1HOH6iR+k3s7FBtfDuQnNzc7S0tGRdxqB++avdvPPvH+ILVy7l8jeP+WcIzGyMk7QuIpoH6zfYGUnvzjbw+ktcLwEtwGd6zyKsMkvnTaNpynhWt253kJjZmFFWkFC8fNQN9L55fhUwEXiB4mWm3xr2ympQXZ24ZMksVq5/jkNd3YxvqM+6JDOzQZX7+O8lEXFjRGxIlk8Cb4uIvwAWpFde7Vmen83Lh7t56Emf5JnZ2FBukNRLWtb7RdI5FG+gA3QNe1U17K0nn8DExnq/nGhmY0a5QfJh4CuSnpL0NPAV4MOSJgH/J63iatGEcfW8bXETd7dup6en+h+EMLOxr9wXEtdGxBkU52o/KyLOTNpejojb0y2x9hTyOTr2HuLRbS9lXYqZ2aDKHUb+eEmfA+4B7pH0fyUdn25pteui02ZRXyfPUWJmY0K5l7a+CuwFrkiWPcDX0iqq1k2b2Mg5CzyIo5mNDeUGyckR8alkSPj2iPhT4KQ0C6t1hfxstmzfx9M7Xs66FDOzAZUbJAcknd/7RdJ5wIF0SjIoTnYFnqPEzEa/coPkf1Ccr/1pSc8AXwT+e3pl2fwZEzlt9hQHiZmNemW92R4R64GzJE1NvntO2BGwPJ/ji/e1sevlw8yY1Jh1OWZm/RpsGPmPHaUdgIj4XAo1WaKQn83f3tvGPZu289vN8wffwMwsA4Nd2poyyGIpetPcqcw5foIvb5nZqDbgGUnydJZlRBKXLMnxL+ue5cDhbo5r9CCOZjb6lHuz/RWSfplGIda/5afnOHikh5+27ci6FDOzfh1zkAAa9irsqM5deAJTxjf4LXczG7WGEiQ/HPYq7KgaG+q48LRZ3LOpg24P4mhmo9AxB0lE/HG5fSWtkLRZUpukG/pZf42kTknrk+XDSfvbS9rWSzoo6fJk3deTUYh71y091mMYawr5HDtfPsx//Gp31qWYmb1OuVPt7uXoU+3+YUS097NNPXAzUAC2AmslrYyI1j5dvxsR15U2RMR9FEcaRtIMoA24q6TLxyPijnJqrwYXntrEuHqxunU7zQtmZF2OmdlrlHtG8gXg48BcYB7wPylOu3sbxQEd+7MMaEvG5jqc9L1sCDW+G/hxROwfwrZVYeqEcbzlpBO4q3U7Eb68ZWajS7lBcmlEfCki9kbEnoi4BfjNiPguMP0o28wFni35vjVp6+tdkh6VdIek/t66uwr4Tp+2zybbfF7S+DKPYUxbns/x1I6XebJzX9almJm9RrlBsl/SFZLqkuUK4GCyrpJ/Rf4BsCAizgRWA98oXSlpDnAGsKqk+UbgNOAcYAbwif52LOlaSS2SWjo7OysocXS4JBnE8S6/nGhmo0y5QfJe4HeADmB78vl9ko4DrjvKNtuA0jOMeUnbKyJiZ0QcSr7eCpzdZx9XAHdGxJGSbZ6PokMU50RZRj8i4paIaI6I5qampnKOcVSbc/xxnDH3eL/lbmajTrlT7bZHxG9FxMyIaEo+t0XEgYj46VE2WwsskrRQUiPFS1QrSzskZxy9LgU29dnH1fS5rNW7jYoDfl0OPFbOMVSDQj7H+mdfpGPvwcE7m5mNkHKn2l0s6R5JjyXfz5Q04GPAEdFF8WxlFcWAuD0iNkq6SdKlSbfrJW2U9AhwPXBNyW8uoHhG80CfXX9L0gZgAzAT+Ew5x1ANCvkcEXDPpo6sSzEze4XKeQpI0gMUn9r6UkS8OWl7LCLelHJ9w6K5uTlaWlqyLqNiEcFv/OV9LM5N4avXnJN1OWZW5SSti4jmwfqVe49kYkT8ok9b17GXZZWQRCGf46dtO3j5kP/rN7PRodwg2SHpZJIntCS9G3g+tarsqJbnZ3O4q4cHt4z9J9HMrDqUGyS/D3wJOE3SNuCjeKrdTJyzYDrTJo7z01tmNmqUNUQKxcd2vwbcR/HdjT3AB4CbUqrLjqKhvo6LTp3FvZs76OruoaF+KONumpkNn3L/Cn0f+C3gCPAcsA94Oa2ibGCFfI4X9x9h7dMexNHMslfuGcm8iFiRaiVWtgsWN9HYUMfq1u289eQTsi7HzGpcuWckD0k6I9VKrGyTxjdw3skncFfrCx7E0cwyV26QnA+sS+YWeVTSBkmPplmYDWz56bPZuvsAj7+wN+tSzKzGlXtp6x2pVmHH7OIls5Bgdet2lsyZmnU5ZlbDyh1r65n+lrSLs6ObNWUCS+dP82PAZpY5Pzs6hhXyOTZse4nnXzqQdSlmVsMcJGPY8mSOkrt9VmJmGXKQjGEnN01m4cxJnuzKzDLlIBnDJLE8n2NN+072HDwy+AZmZilwkIxxhXyOI93B/Zs9iKOZZcNBMsa9+Y3TOWFSo5/eMrPMOEjGuPo6cfGSWdz/eAeHu3qyLsfMapCDpAoU8rPZe6iLh5/amXUpZlaDUg0SSSuSYVXaJN3Qz/prJHVKWp8sHy5Z113SvrKkfaGkh5N9fldSY5rHMBacf8pMJoyr466NvrxlZiMvtSCRVA/cTHF4lTxwtaR8P12/GxFLk+XWkvYDJe2XlrT/BfD5iDgF2A18KK1jGCuOa6znNxY1cfem7R7E0cxGXJpnJMuAtohoj4jDwG3AZZXsUJKAi4A7kqZvAJdXVGWVWJ7P8fxLB3ls256sSzGzGpNmkMwFni35vjVp6+tdyYjCd0iaX9I+QVKLpDWSesPiBODFiOgaZJ9IujbZvqWzs/ofjb14SY46werWF7IuxcxqTNY3238ALIiIM4HVFM8wep0YEc3Ae4AvSDr5WHYcEbdERHNENDc1NQ1fxaPUjEmNNJ84w2+5m9mISzNItgGlZxjzkrZXRMTOiDiUfL0VOLtk3bbkn+3A/cCbgZ3ANEm9w9+/bp+1rJDP8fgLe3l21/6sSzGzGpJmkKwFFiVPWTUCVwErSztImlPy9VJgU9I+XdL45PNM4DygNYp3ku8D3p1s8wGK88kbxSAB/HKimY2o1IIkuY9xHbCKYkDcHhEbJd0kqfcprOslbZT0CHA9cE3SvgRoSdrvA/48IlqTdZ8APiapjeI9k6+kdQxjzYKZk1g0azJ3+T6JmY2gcmdIHJKI+BHwoz5tf1Ly+Ubgxn62ewjod4745FLXsuGttHosPz3HPz7Qzov7DzNtYs2/YmNmIyDrm+02zAr52XT3BPc+3pF1KWZWIxwkVebMuccza8p43ycxsxHjIKkydXXiknyOB7Z0cvBId9blmFkNcJBUoUI+x/7D3fz8SQ/iaGbpc5BUoV8/+QQmNdb76S0zGxEOkio0vqGeC0+dxd2bOujp8SCOZpYuB0mVKuRzdO49xPqtL2ZdiplVOQdJlXr7qbOor5Of3jKz1DlIqtTxE8dx7sIZDhIzS52DpIoV8jnaOvbR3rkv61LMrIo5SKqYB3E0s5HgIKli86ZPJD9nqoPEzFLlIKlyhXyOdb/azY59hwbvbGY2BA6SKlfI54iAezd5EEczS4eDpMqd/oapzJ12nKfgNbPUOEiqnCQuWTKLn7Z1cuCwB3E0s+HnIKkBhfxsDh7p4SdPdGZdiplVoVSDRNIKSZsltUm6oZ/110jqlLQ+WT6ctC+V9PNkGt5HJV1Zss3XJT1Vss3SNI+hGpx70gymTGjw5S0zS0VqU+1KqgduBgrAVmCtpJUlc6/3+m5EXNenbT/w/oh4QtIbgHWSVkVE78BRH4+IO9KqvdqMq6/jotNmce/jHXT3BPV1yrokM6siaZ6RLAPaIqI9Ig4DtwGXlbNhRGyJiCeSz88BHUBTapXWgEI+x66XD7Pumd1Zl2JmVSbNIJkLPFvyfWvS1te7kstXd0ia33elpGVAI/BkSfNnk20+L2n8sFZdpd62uIlx9WK15ygxs2GW9c32HwALIuJMYDXwjdKVkuYA/wR8MCJ6kuYbgdOAc4AZwCf627GkayW1SGrp7PRN5ikTxvHWk2eyunU7EZ6jxMyGT5pBsg0oPcOYl7S9IiJ2RkTvK9e3Amf3rpM0Ffgh8MmIWFOyzfNRdAj4GsVLaK8TEbdERHNENDc1+aoYFC9vPb1zP20dHsTRzIZPmkGyFlgkaaGkRuAqYGVph+SMo9elwKakvRG4E/hm35vqvdtIEnA58FhqR1BlCkuKgzj66S0zG06pBUlEdAHXAasoBsTtEbFR0k2SLk26XZ884vsIcD1wTdJ+BXABcE0/j/l+S9IGYAMwE/hMWsdQbWYfP4Gz5h3vIDGzYaVauF7e3NwcLS0tWZcxKnzx3if467u28PD/upjc1AlZl2Nmo5ikdRHRPFi/rG+22wgr5GcDcPcmn5WY2fBwkNSYxbnJvHHGRM9RYmbDxkFSYyRRyOd4qG0n+w51ZV2OmVUBB0kNKuRzHO7u4YHNfr/GzCrnIKlBzSdOZ/rEcX7L3cyGhYOkBjXU13HRaTnufbyDI909g29gZjYAB0mNKuRz7DnYxdqndmVdipmNcQ6SGnXB4pmMb6jzy4lmVjEHSY2a2NjA+ad4EEczq5yDpIYV8jm2vXiA1uf3ZF2KmY1hDpIadvGSHBJ+OdHMKuIgqWFNU8bza2+c7iAxs4o4SGpcIZ9j43N72PbigaxLMbMxykFS4wr54hwld/usxMyGyEFS405umsxJTZN8ecvMhsxBYhTyOda07+SlA0eyLsXMxiAHibE8n6OrJ7h/c0fWpZjZGOQgMZbOn87MyeP9lruZDUmqQSJphaTNktok3dDP+mskdZbMy/7hknUfkPREsnygpP1sSRuSff6tJKV5DLWgvk5csmQWD2zu5FBXd9blmNkYk1qQSKoHbgbeAeSBqyXl++n63YhYmiy3JtvOAD4FnAssAz4laXrS/x+AjwCLkmVFWsdQSwr5HPsOdbGm3YM4mtmxSfOMZBnQFhHtEXEYuA24rMxtfxNYHRG7ImI3sBpYIWkOMDUi1kRxgKhvApenUXytOe+UmRw3rt5zlJjZMUszSOYCz5Z835q09fUuSY9KukPS/EG2nZt8HmyfSLpWUoukls5OzwQ4mAnj6rlg8Uzubu2gp8eDOJpZ+Roy/v0fAN+JiEOS/hvwDeCi4dhxRNwC3ALQ3Nzsv4xlKORns2rjdjZse4mz5k/Luhw7BhFBV09wpLuHI13B4e6e4udkOdwVJZ97ONzdQwAERPETEclSss9XPyed6dunuH3vANKRbMdrtuvTJ17br2TX/eyr/3b61Fa6r8FqGYpXf22I21f025W5snk+0yc1VriXgaUZJNuA+SXf5yVtr4iInSVfbwX+smTbC/tse3/SPm+gfdrQXXzaLOoEf/L9xzhl1hTq66C+rq74T+nVzwO11dVRr+IN/L7r6iQa+u0v6up47bqjtNXX67Xr+mmrEwzXMxhD+QN9pPu1ba/5/sp+ujnSHcn63iXZf1fJ91f2+dq2123T3VPRHyurXpcsyY3pIFkLLJK0kOIf+6uA95R2kDQnIp5Pvl4KbEo+rwL+rOQG+3LgxojYJWmPpLcADwPvB/4uxWOoKdMnNfLec0/k/i0drGnfSVdPD9090BNBV3cPPQFdPT309EB3BN2j+BJYfd3rw6Whvi4JM1Ff99p1PcGI/oGurxPj6kVjfR2NDXWMq+9dxLjXtIkp4xoY/5o+dTQ26LXfk+3GNdTR+Mo/NcA2oi4JW0n0xq4EQpTmcO/n0vbefq9+pmSbkn68Guo6yr5e2ccg/fTKf7za/mrdrz+G3o36q2UoKv1Xk0r+3UYV/Pr4hvTf8kgtSCKiS9J1FEOhHvhqRGyUdBPQEhErgeslXQp0AbuAa5Jtd0n6NMUwArgpInofJ/o94OvAccCPk8WGyacvf1PZfSPi9eHSHXRH9NvWnQRTd08xhIbS1tNTPEMo7reH7ni1rRh4xXV923qS4Ovqee263t8o/mE/+h/oV/7tbliGAAAFCUlEQVSwD/oHuq7kj/7rQ6G3T32dn1q36qFamB2vubk5Wlpasi7DzGxMkbQuIpoH6+c3283MrCIOEjMzq4iDxMzMKuIgMTOzijhIzMysIg4SMzOriIPEzMwq4iAxM7OK1MQLiZI6gWeGuPlMYMcwljMW+Jhrg4+5+lV6vCdGRNNgnWoiSCohqaWcNzuriY+5NviYq99IHa8vbZmZWUUcJGZmVhEHyeBuybqADPiYa4OPufqNyPH6HomZmVXEZyRmZlYRB0kZJP2VpMclPSrpTklVP6G5pN+WtFFSj6SqfcpF0gpJmyW1Sboh63pGgqSvSuqQ9FjWtYwESfMl3SepNfm/6T/Iuqa0SZog6ReSHkmO+U/T/D0HSXlWA2+KiDOBLcCNGdczEh4D3gk8mHUhaZFUD9wMvAPIA1dLymdb1Yj4OrAi6yJGUBfwhxGRB94C/H4N/O98CLgoIs4ClgIrkinKU+EgKUNE3BURXcnXNcC8LOsZCRGxKSI2Z11HypYBbRHRHhGHgduAyzKuKXUR8SDFqa1rQkQ8HxG/TD7vBTYBc7OtKl1RtC/5Oi5ZUrsh7iA5dr+L54mvFnOBZ0u+b6XK/8DUOkkLgDcDD2dbSfok1UtaD3QAqyMitWNuSGvHY42ku4HZ/az6ZER8P+nzSYqnyd8aydrSUs4xm1ULSZOBfwU+GhF7sq4nbRHRDSxN7uneKelNEZHKfTEHSSIiLhlovaRrgP8CXBxV8sz0YMdcA7YB80u+z0varMpIGkcxRL4VEf+WdT0jKSJelHQfxftiqQSJL22VQdIK4I+ASyNif9b12LBZCyyStFBSI3AVsDLjmmyYSRLwFWBTRHwu63pGgqSm3qdLJR0HFIDH0/o9B0l5vghMAVZLWi/pH7MuKG2S/qukrcBbgR9KWpV1TcMteYDiOmAVxRuwt0fExmyrSp+k7wA/B06VtFXSh7KuKWXnAb8DXJT8/+96Sf8p66JSNge4T9KjFP+FaXVE/HtaP+Y3283MrCI+IzEzs4o4SMzMrCIOEjMzq4iDxMzMKuIgMTOzijhIzIaBpH2D9xpw+zsknZR8nizpS5KelLRO0v2SzpXUKOlBSX6R2EYVB4lZxiSdDtRHRHvSdCvFQRUXRcTZwAeBmcnAkvcAV2ZTqVn/HCRmw0hFfyXpMUkbJF2ZtNdJ+vtkXpvVkn4k6d3JZu8FesdzOxk4F/jjiOgBiIinIuKHSd/vJf3NRg2fIpsNr3dSnP/hLGAmsFbSgxTfrl5Acd6TWRTfpP9qss15wHeSz6cD65MB9/rzGHBOKpWbDZHPSMyG1/nAdyKiOyK2Aw9Q/MN/PvAvEdETES8A95VsMwfoLGfnScAcljRlmOs2GzIHiVn2DgATks8bgbOS2RuPZjxwMPWqzMrkIDEbXj8BrkwmFWoCLgB+AfwMeFdyryQHXFiyzSbgFICIeBJoAf40GbUWSQsk/efk8wnAjog4MlIHZDYYB4nZ8LoTeBR4BLgX+KPkUta/UpyBsRX4Z+CXwEvJNj/ktcHyYSAHtEl6jOIc6x3Jurcn/c1GDY/+azZCJE2OiH3JWcUvgPMi4oVkvoj7ku9Hu8neu49/A26IiC0jULJZWfzUltnI+fdksqFG4NPJmQoRcUDSpyjOF/+ro22cTL71PYeIjTY+IzEzs4r4HomZmVXEQWJmZhVxkJiZWUUcJGZmVhEHiZmZVcRBYmZmFfn/sSlUjggD3wwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 1\n",
    "scores = np.zeros((n_classes, n_Cs))\n",
    "scores[0][:] = np.mean(lrcv_L1.scores_[1], axis=0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.02397919,  0.36824092,  0.97810163, -0.09004296,  0.        ,\n",
       "         0.        ,  0.67771259,  0.27742566,  0.10733749]])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49670404390360395\n"
     ]
    }
   ],
   "source": [
    "print(min(mse_mean))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.C_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='liblinear', tol=0.0001,\n",
       "           verbose=0)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [0.001,0.01,0.1,1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1.0: array([[-0.63333908, -0.50893127, -0.45696888, -0.45462132, -0.45482561,\n",
       "         -0.45485388, -0.45485678],\n",
       "        [-0.64753809, -0.56138381, -0.53112167, -0.53063466, -0.53084305,\n",
       "         -0.53086817, -0.53087073],\n",
       "        [-0.64956908, -0.56700403, -0.55304073, -0.56498889, -0.56732711,\n",
       "         -0.5675809 , -0.56760649],\n",
       "        [-0.64064668, -0.5439538 , -0.50791888, -0.5049356 , -0.50481832,\n",
       "         -0.50480998, -0.50480918],\n",
       "        [-0.63045642, -0.49832736, -0.43928609, -0.43343586, -0.43307585,\n",
       "         -0.43304404, -0.4330409 ]])}"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3XmcFfWd7vHP0xs7gtAiDSguEAWFjrao0RizGTSxexIzLnE3xJvJ9TpJJpkxN3OvE53JnSyTSebGzETRRGOMek0iEE0QE43GRKUxgAIuiEZZxAZUZO/le/84BR7bhnNourq6Tz/v16tenqr61TnfUqjH2n4/RQRmZmZ7UpZ1AWZm1vM5LMzMrCCHhZmZFeSwMDOzghwWZmZWkMPCzMwKcliYmVlBDgszMyvIYWFmZgVVZF1AVxk5cmSMHz8+6zLMzHqVBQsWrIuI6kLtSiYsxo8fT2NjY9ZlmJn1KpL+Ukw7X4YyM7OCHBZmZlaQw8LMzApyWJiZWUEOCzMzKyjVsJA0XdIzkpZLumo3bc6WtFTSEkm3tVs3VNJKSd9Ps04zM9uz1B6dlVQOXAd8GFgJzJc0OyKW5rWZAHwFOCkiXpN0QLuvuRZ4KK0azcysOGmeWUwDlkfEiojYAdwONLRr8xnguoh4DSAiXt25QtKxwCjgvhRr5PUtO/je/c+xbM3GNH/GzKxXSzMsxgAv582vTJblmwhMlPSIpEclTQeQVAb8G/ClFOsDQIjvP/AcP1+wMu2fMjPrtbK+wV0BTABOBc4DbpA0DPgccG9E7PEILulySY2SGpuamjpVwH4DK3nfxAOYs3g1rW3Rqe8wMyt1aYbFKmBc3vzYZFm+lcDsiGiOiBeAZ8mFx4nAFZJeBL4NXCTpX9v/QERcHxF1EVFXXV2wa5PdaqitYe3G7Tz2wvpOf4eZWSlLMyzmAxMkHSKpCjgXmN2uzd3kziqQNJLcZakVEXF+RBwUEePJXYq6JSI6fJqqK3zoyFEMrCpn9sLVaf2EmVmvllpYREQLcAUwF1gG3BkRSyRdI6k+aTYXWC9pKfAA8OWI6Pb/vR9QVc5HJh/Ir596he0trd3982ZmPZ4iSuM6fV1dXexLr7MPPP0ql/54PjdcVMeHJ43qwsrMzHouSQsioq5Qu6xvcPcYJ08YyfCBlcxa2P62ipmZOSwSleVlfHTKaO5ftpbN21uyLsfMrEdxWORpqB3DtuY25i1dm3UpZmY9isMiz7EHDWfMsAG+FGVm1o7DIk9ZmfjY1NE89Nw61m/annU5ZmY9hsOinYapY2htC+596pWsSzEz6zEcFu0cOXoIEw4YzGxfijIz28Vh0Y4kGmprmP/ia6x6fWvW5ZiZ9QgOiw6cObUGgDmL3P2HmRk4LDp08IhB1I4bxiz3FWVmBjgsdquhtoZlazby3No3sy7FzCxzDovd+OiU0ZQJZvtSlJmZw2J3DhjSn/ccNpJZC1dTKp0tmpl1lsNiD+pra3hpwxYWvvx61qWYmWXKYbEH0486kKqKMt/oNrM+z2GxB0P7V/KBdx3ArxavoaW1LetyzMwy47AooKG2hnWbtvPoig1Zl2JmlhmHRQHvP+IABvercE+0ZtanOSwK6F+ZG5/7N0+9wrZmj89tZn1TqmEhabqkZyQtl3TVbtqcLWmppCWSbkuW1Ur6U7JssaRz0qyzkIbaGt7c3sKDz7yaZRlmZplJLSwklQPXAacDk4DzJE1q12YC8BXgpIiYDHw+WbUFuChZNh34rqRhadVayHsOG8HIwVV+Qc/M+qw0zyymAcsjYkVE7ABuBxratfkMcF1EvAYQEa8m/3w2Ip5LPq8GXgWqU6x1jyrKy/jo0aO5f9mrvLmtOasyzMwyk2ZYjAFezptfmSzLNxGYKOkRSY9Kmt7+SyRNA6qA5ztYd7mkRkmNTU1NXVj6O9XXjmFHSxtzl3h8bjPre7K+wV0BTABOBc4Dbsi/3CRpNPAT4NKIeMeLDhFxfUTURURddXW6Jx7HHDSMscM9PreZ9U1phsUqYFze/NhkWb6VwOyIaI6IF4BnyYUHkoYC9wBfjYhHU6yzKDsHRfrj8+tpetPjc5tZ35JmWMwHJkg6RFIVcC4wu12bu8mdVSBpJLnLUiuS9r8EbomIu1Ksca801Cbjcz+5JutSzMy6VWphEREtwBXAXGAZcGdELJF0jaT6pNlcYL2kpcADwJcjYj1wNnAKcImkhclUm1atxZo4aghHHDjEl6LMrM9RqXS/XVdXF42Njan/zg8eXM43f/MMD335/Rw0YmDqv2dmliZJCyKirlC7rG9w9zpnTknG517sdy7MrO9wWOylcfsPpO7g4cx2t+Vm1oc4LDqhvraGZ9a+ydOvbMy6FDOzbuGw6IQzjh5NeZk8KJKZ9RkOi04YObgfJx8+ktken9vM+giHRSc11Naw6vWtPPHSa1mXYmaWOodFJ502+UD6eXxuM+sjHBadNLhfBR86chT3LF5Ds8fnNrMS57DYB/W1NazfvINHlq/LuhQzs1Q5LPbBqe+qZkj/Cg+KZGYlz2GxD/pVlHPGUaOZ6/G5zazEOSz2UX1tDZt3tPLbZR6f28xKl8NiH51w6Aiqh/RzT7RmVtIcFvuovEycOaWGB59p4o2tHp/bzEqTw6ILNNTWsKO1jblPvZJ1KWZmqXBYdIEpY/fj4BEDmbXIl6LMrDQ5LLqAJBqm5sbnfnXjtqzLMTPrcg6LLlJfW0MEzFns8bnNrPSkGhaSpkt6RtJySVftps3ZkpZKWiLptrzlF0t6LpkuTrPOrnD4AUOYXDPUL+iZWUlKLSwklQPXAacDk4DzJE1q12YC8BXgpIiYDHw+Wb4/cDVwPDANuFrS8LRq7SoNtTUsevl1Xly3OetSzMy6VJpnFtOA5RGxIiJ2ALcDDe3afAa4LiJeA4iInW+2fQSYFxEbknXzgOkp1tolPpaMz+2zCzMrNWmGxRjg5bz5lcmyfBOBiZIekfSopOl7sW2PUzNsANMO2Z9ZC1d5UCQzKylZ3+CuACYApwLnATdIGlbsxpIul9QoqbGpqSmlEvdOQ20NzzdtZukaj89tZqUjzbBYBYzLmx+bLMu3EpgdEc0R8QLwLLnwKGZbIuL6iKiLiLrq6uouLb6zzjhqNBVlYrYHRTKzEpJmWMwHJkg6RFIVcC4wu12bu8mdVSBpJLnLUiuAucBpkoYnN7ZPS5b1eMMHVXHKxGpmL1pNW5svRZlZaUgtLCKiBbiC3EF+GXBnRCyRdI2k+qTZXGC9pKXAA8CXI2J9RGwAriUXOPOBa5JlvUJDbQ1r3tjG/Bd7TclmZnukUrkRW1dXF42NjVmXAcDm7S3U/fP9fPyYMXz940dnXY6Z2W5JWhARdYXaZX2DuyQN6lfBhyeN4t4n17CjxeNzm1nv57BISf3UGl7f0swflveMp7TMzPaFwyIlp0ysZr8BlczyU1FmVgIcFimpqijjjKNHc9+StWzZ0ZJ1OWZm+8RhkaKG2hq2Nrdyv8fnNrNezmGRomnj9+fAof2Z7fG5zayXc1ikqKxMnDl1NA8+08Rrm3dkXY6ZWac5LFLWUDuGlrbg1x6f28x6MYdFyibXDOXQ6kHM9vjcZtaLOSxSlhufewyPvbCBNW9szbocM7NOcVh0g53jc/9qkcfnNrPeyWHRDQ4ZOYgpY/djli9FmVkvVVRYSDpJ0qDk8wWSviPp4HRLKy31U2t4atVGnm/alHUpZmZ7rdgzi/8EtkiaCvwd8DxwS2pVlaAzp9Yg4UGRzKxXKjYsWiLXl3kD8P2IuA4Ykl5ZpWfU0P6ceOgIZi9a7fG5zazXKTYs3pT0FeAC4B5JZUBlemWVpvqpNbywbjNPrnoj61LMzPZKsWFxDrAd+HREvEJuTOxvpVZViTr9qNFUlss90ZpZr1P0mQXwvYh4WNJEoBb4WXpllab9BlZy6rsO4FeLV9Pq8bnNrBcpNiweAvpJGgPcB1wI/DitokpZQ20Nazdu57EX1mddiplZ0YoNC0XEFuATwA8i4q+BowpuJE2X9Iyk5ZKu6mD9JZKaJC1Mphl5674paYmkZZL+Q5KK3ame7INHjGJQVbmfijKzXqXosJB0InA+cE8x20oqB64DTgcmAedJmtRB0zsiojaZZibbvgc4CZhCLpSOA95XZK092oCqck6bfCD3PrmG7S2tWZdjZlaUYsPi88BXgF9GxBJJhwIPFNhmGrA8IlZExA7gdnKP3hYjgP5AFdCP3JNXa4vctserr61h47YWHnp2XdalmJkVpaiwiIjfR0Q9cJ2kwUkAXFlgszHAy3nzK5Nl7Z0labGkuySNS37vT+TCaE0yzY2IZcXU2hucfPhI9h9UxSwPimRmvUSx3X0cLenPwBJgqaQFkiZ3we/PAcZHxBRgHnBz8nuHA0eSe0R3DPABSe/toK7LJTVKamxqauqCcrpHZXkZZxx9IPcvW8um7R6f28x6vmIvQ/0Q+GJEHBwRB5Hr8uOGAtusAsblzY9Nlu0SEesjYnsyOxM4Nvn8ceDRiNgUEZuAXwMntv+BiLg+Iuoioq66urrIXekZGmrHsK25jXlLPSiSmfV8xYbFoIjYdY8iIh4EBhXYZj4wQdIhkqqAc4HZ+Q0kjc6brQd2Xmp6CXifpApJleRubpfMZSiAYw8azphhA/yCnpn1CsWGxQpJ/0vS+GT6R2DFnjaIiBbgCmAuuQP9ncnN8Wsk1SfNrkwej10EXAlckiy/i1xnhU8Ci4BFETFnr/ash8uNz13Dw8+tY/2m7YU3MDPLkIrp1E7ScOBrwMnJooeBf4qI11Ksba/U1dVFY2Nj1mXslWVrNnL69x7m2r86igtPcI/vZtb9JC2IiLpC7SqK+bIkFAo9/WR76YgDhzDhgMHMXrjKYWFmPdoew0LSHHLvPHQoeZzWOkkSDbU1fPu+Z1n1+lbGDBuQdUlmZh0qdGbx7W6pog+rnzqGb9/3LHMWreaz7zss63LMzDq0x7CIiN93VyF91UEjBvLug4Yxa6HDwsx6rmJfynsyecs6f3pY0r9LGpF2kaWufmoNy9Zs5Nm1b2ZdiplZh4p9dPbX5DoQPD+Z5gCNwCu4q/J99tEpoynz+Nxm1oMV9TQU8KGIOCZv/klJT0TEMZIuSKOwvuSAIf056fCRzF60mr87bSIl0hu7mZWQYs8syiVN2zkj6TigPJl150ZdoH5qDS9t2MLCl1/PuhQzs3coNixmADdKekHSi8CNwAxJg4D/k1ZxfclHjjqQqooyd/9hZj1SsV2Uz4+Io8mNvT01IqYkyzZHxJ3pltg3DO1fyQfedQC/WryGlta2rMsxM3ubYp+G2k/Sd4DfAr+V9G+S9ku3tL6nobaGdZu286cVHp/bzHqWYi9D3QS8CZydTBuBH6VVVF/1/iMOYEi/Cj8VZWY9TrFhcVhEXJ2MkLciIr4GHJpmYX1R/8pyPnLUgfzmqVfY1uzxuc2s5yg2LLZK2tnjLJJOAramU1LfVj+1hje3t/DgM69mXYqZ2S7FvmfxN8DNyX0KARt4a+wJ60LvOWwEIwdXMWvhaqYfNbrwBmZm3aDYLsoXAlMlDU3mN6ZaVR9WUV7Gx6bUcNvjL/HmtmaG9K/MuiQzs4JdlH9xN8sBiIjvpFBTn1dfW8OP//gic5es5ZPHjs26HDOzgvcshhSYLAXvHjeMcfsPYNbCVVmXYmYGFO6i/GvdVYi9RRL1U2v4zwefp+nN7VQP6Zd1SWbWxxX7NNQukp7Yi7bTJT0jabmkqzpYf4mkJkkLk2lG3rqDJN0naZmkpZLG722tvVlD7RjaAu5Z7HcuzCx7ex0W5J6GKtxIKgeuA04HJgHnSZrUQdM7IqI2mWbmLb8F+FZEHAlMA/rUs6QTRw3hiAOHMHuRw8LMsteZsLinyHbTgOXJS3w7gNuBhmI2TEKlIiLmAUTEpojY0olae7WG2jE88dLrvLS+z+26mfUwex0WEfGPRTYdA7ycN78yWdbeWcnIe3dJGpcsmwi8LukXkv4s6VvJmUqfcubU3HsWc3wpyswyVmxHgm9K2thuelnSLyXtS7cfc4DxETEFmAfcnCyvAN4LfAk4jlzXIpd0UNflkholNTY1Ne1DGT3T2OEDqTt4uJ+KMrPMFXtm8V3gy+TODMaSO4jfRu7S0k272WYVMC5vfmyybJeIWB8R25PZmcCxyeeVwMLkElYLcDeQP1Lfzu2vj4i6iKirrq4ucld6l4baGp5du4mnX/F7kGaWnWLDoj4ifhgRb0bExoi4HvhIRNwBDN/NNvOBCZIOkVQFnAvMzm8gKb8/i3pgWd62wyTtTIAPAEuLrLWknHH0aMrL5EGRzCxTxYbFFklnSypLprOBbcm66GiD5IzgCmAuuRC4MyKWSLpGUn3S7EpJSyQtAq4kudQUEa3kzl5+K+lJck9g3dCJ/ev1Rgzux8mHj2T2wtW0tXX4r9rMLHWKKHwASu5LfA84kVw4PAp8gdxlpWMj4g9pFlmMurq6aGxszLqMVPziiZV88c5F3PXZE6kbv3/W5ZhZCZG0ICLqCrUrtiPBFcCZu1mdeVCUutMmH0i/iieZvWi1w8LMMlHs01ATJf1W0lPJ/BRJxT5Ca/tocL8KPjRpFPcsXkOzx+c2swwUe8/iBuArQDNARCwmd8PauknD1BrWb97BI8vXZV2KmfVBxYbFwIh4vN2ylq4uxnbvfe+qZmh/j89tZtkoNizWSTqM5MknSZ8E1qRWlb1Dv4pyTj9qNHOXvMLWHR6f28y6V7Fh8d+BHwJHSFoFfB74bGpVWYcaamvYvKOV3z3dp/pUNLMeoNiwWAX8CPgXcm9tzwMuTqso69jxh47ggCH93P2HmXW7YsNiFrlHZ5uB1cAmYHNaRVnHysvEx6bU8OAzTbyxpTnrcsysDynqPQtgbERMT7USK0pDbQ03PfICv1myhnOOOyjrcsysjyj2zOKPko5OtRIrypSx+zF+xEAPimRm3arYsDgZWJAMkbpY0pOSFqdZmHVMEvW1Y/jj8+t5deO2whuYmXWBYsPidGACcBq5excfY/fdf1jK6qfWEAFzFvvpZTPrHkWFRUT8paMp7eKsY4cfMJjJNUOZ7aeizKybdGYMbusBGmprWLTyDV5c54fSzCx9Dote6sypNUj4RreZdQuHRS81er8BTBu/P3cvXEUxY5KYme0Lh0UvVl9bw4qmzSxZ7fG5zSxdDote7IyjRlNRJl+KMrPUOSx6seGDqnjfxGrmLPL43GaWrlTDQtL05EW+5ZKu6mD9JZKaJC1Mphnt1g+VtFLS99Osszerr61hzRvbmP/ihqxLMbMSllpYSCoHriP3Qt8k4DxJkzpoekdE1CbTzHbrrgUeSqvGUvDhSaMYUFnOLF+KMrMUpXlmMQ1YHhErImIHua7NG4rdWNKxwCjgvpTqKwkDqyr48KRR3PvkGna0eHxuM0tHmmExBng5b35lsqy9s5L+pu6SNA5AUhnwb8CX9vQDki6X1Cipsampqavq7nUaamt4fUszf1jed/8dmFm6sr7BPQcYHxFTyA2odHOy/HPAvRGxck8bR8T1EVEXEXXV1dUpl9pzvXdCNcMGVjLL43ObWUqKHc+iM1YB4/LmxybLdomI9XmzM4FvJp9PBN4r6XPAYKBK0qaIeMdNcoOqijLOOHo0v3xiFVt2tDCwKs3/rGbWF6V5ZjEfmCDpEElVwLnA7PwGkkbnzdYDywAi4vyIOCgixpO7FHWLg2LP6qfWsLW5lXlL12ZdipmVoNTCIiJagCuAueRC4M6IWCLpGkn1SbMrJS2RtAi4ErgkrXpK3bTx+3Pg0P7M8VNRZpYClUq/QnV1ddHY2Jh1GZn6+r3LuOkPL3DfF07h0OrBWZdjZr2ApAURUVeoXdY3uK0LXXjCwQwdUMkFMx9j5Wtbsi7HzEqIw6KEjNt/ID/59DQ2bW/hUzc8xloPu2pmXcRhUWIm1+zHzZdNY/2m7Zw/8zHWb9qedUlmVgIcFiXo3QcN58ZLjmPla1u44MbHeWNLc9YlmVkv57AoUSccOoIfXljH869u4uIfPc6m7S1Zl2RmvZjDooS9b2I13//Uu3ly1Rtc9uP5bN3RmnVJZtZLOSxK3GmTD+Q7Z09l/osbuPwnjWxvcWCY2d5zWPQBDbVj+MYnpvDwc+u44rY/09zq3mnNbO84LPqIs48bx9fqJzNv6Vq+eOciWj2ynpntBfc414dc/J7xbNnRyjd+8zQDKsv4109MoaxMWZdlZr2Aw6KP+ZtTD2Prjhb+43fLGVBZzj/VT0ZyYJjZnjks+qAvfHgiW3a0MvMPLzCgqoJ/mP4uB4aZ7ZHDog+SxFc/eiRbm1v5r98/z8Cqcq784ISsyzKzHsxh0UdJ4tqGo9ja3Mp35j3LwKpyZrz30KzLMrMeymHRh5WViW+eNYXtzW388z3L6F9ZzgUnHJx1WWbWAzks+riK8jL+/Zxatja38o93P8WAynLOOnZs1mWZWQ/j9yyMqooyfnD+MZx0+Ai+fNci7n1yTdYlmVkP47AwAPpXlnPDRXUcc9BwrvzZn/nd0x7L28zekmpYSJou6RlJyyVd1cH6SyQ1SVqYTDOS5bWS/pSMz71Y0jlp1mk5A6squOnS45hUM5TP3voEjyxfl3VJZtZDpBYWksqB64DTgUnAeZImddD0joioTaaZybItwEURMRmYDnxX0rC0arW3DO1fyc2XTuOQEYOYcXMjjS9uyLokM+sB0jyzmAYsj4gVEbEDuB1oKGbDiHg2Ip5LPq8GXgWqU6vU3mb4oCpunXE8o/frz6U/ms/ila9nXZKZZSzNsBgDvJw3vzJZ1t5ZyaWmuySNa79S0jSgCng+nTKtI9VD+vHTzxzPfgMrueimx3n6lY1Zl2RmGcr6BvccYHxETAHmATfnr5Q0GvgJcGlEvKNfbUmXS2qU1NjU1NQtBfclo/cbwG0zTqB/RTkXzHycFU2bsi7JzDKSZlisAvLPFMYmy3aJiPURsT2ZnQkcu3OdpKHAPcBXI+LRjn4gIq6PiLqIqKuu9lWqNBw0YiC3zjieiOD8mY/x8oYtWZdkZhlIMyzmAxMkHSKpCjgXmJ3fIDlz2KkeWJYsrwJ+CdwSEXelWKMV4fADBnPrjOPZsqOVT818lFfe2JZ1SWbWzVILi4hoAa4A5pILgTsjYomkayTVJ82uTB6PXQRcCVySLD8bOAW4JO+x2tq0arXCjhw9lFsum8Zrm5s5f+ajrNu0vfBGZlYyFFEaI6bV1dVFY2Nj1mWUvMdf2MBFNz3G+BGDuP3yExg2sCrrksxsH0haEBF1hdplfYPbeplph+zPDRfVsaJpMxff9DhvbmvOuiQz6wYOC9tr751QzQ/OP4Ylqzfy6R83smVHS9YlmVnKHBbWKR+aNIrvnltL41828N9+soBtza1Zl2RmKXJYWKd9bEoN3/zkVB5+bh1X3PYEza3veBXGzEqEw8L2ySePHcu1DZO5f9mrfOGOhbS2lcYDE2b2dh78yPbZhSeOZ2tzK1+/92n6V5bzzbOmUFamrMsysy7ksLAucfkph7FlRyvfvf85BlSWc03DZCQHhlmpcFhYl/nbD05g645WfvjQCgZWlXPV6Uc4MMxKhMPCuowkrjr9CLY25wJjQFU5n//QxKzLMrMu4LCwLiWJfzpz8q5LUgOryrn8lMOyLsvM9pHDwrpcWZn4xllT2Jbc9B5QWc6FJ47Puiwz2wcOC0tFeZn493Nq2dbcxv+atYT+leX8dd07xrYys17C71lYairLy/j+p97NeyeM5B9+vpg5i1ZnXZKZdZLDwlLVv7Kc6y+so+7g/fnCHQu5f+narEsys05wWFjqBlSVc+MldUyuGcrnfvoEDz/nIXDNehuHhXWLIf0rufmyaRxaPYjP3NLI4y9syLokM9sLDgvrNsMGVnHrjOOpGTaAy348n4Uvv551SWZWJIeFdauRg/tx24wTGD6okotvepxlazZmXZKZFcFhYd3uwP36c9uMExhYVc4FMx9j+aubsi7JzApINSwkTZf0jKTlkq7qYP0lkpokLUymGXnrLpb0XDJdnGad1v3G7T+QW2ccjwTnz3yUl9ZvybokM9uD1MJCUjlwHXA6MAk4T9KkDpreERG1yTQz2XZ/4GrgeGAacLWk4WnVatk4rHowt844nu0tbXxq5qOseWNr1iWZ2W6keWYxDVgeESsiYgdwO9BQ5LYfAeZFxIaIeA2YB0xPqU7L0BEHDuWWy6bxxpZmzr/hMZre3J51SWbWgTTDYgzwct78ymRZe2dJWizpLkk7+4ModlsrAVPGDuNHlx7Hmje2ceGNj/Ha5h1Zl2Rm7WR9g3sOMD4ippA7e7h5bzaWdLmkRkmNTU1+0as3qxu/PzMvrmPFus1c/KPH2bitOeuSzCxPmh0JrgLye44bmyzbJSLW583OBL6Zt+2p7bZ9sP0PRMT1wPUAdXV1Hvy5lzvp8JH81wXHcPktC7jsR/O55dPTGFhV+n1dtrUFLW1BWwStbUFrxFvLkvnWtqCtjeRzG61t5JYl6/K337nNru13rYOWtrZdn3ff7u2//VYN0BZBRO6vWvIP8v/iRd5MEO9Yxtvaxl5v33Hb6GBZfn27Wd9xWb3SQfsP5MoPTkj1N9L8mzgfmCDpEHIH/3OBT+U3kDQ6ItYks/XAsuTzXODreTe1TwO+kmKt1kN84IhRfO/cd/M/fvYEp37rQYb0z/0R7eC40O7A0YkDTwdHi8hbGB206+jAs7vfJO83dx3IIzlg7zrw945DVnmZKC8TZYKyvNEPd35SB8vyZ/KXva1tofXt2rVv3fH2+S3VwbKOf6s3e2Nr+mfiqYVFRLRIuoLcgb8cuCkilki6BmiMiNnAlZLqgRZgA3BJsu0GSdeSCxyAayLC/UP0ER+dMpqK8mOZvTCvl9oO/rIXOqjs8UDWwQElf/uODj4dtuvgINZRDWXSrgNu7qArysugXKK8rIzystw4IOV6a31Fud7aTqKsTFSUKa8dRbbb3fdBRVkZZbvqePs2+d9nptjdOWIvU1dXF42NjVmXYWbWq0haEBF1hdplfYPbzMx6AYeFmZkV5LAwM7OCHBZmZlaQw8LMzApyWJiZWUEqdKseAAAFTUlEQVQOCzMzK8hhYWZmBZXMS3mSmoC/7MNXjATWdVE5WSqV/QDvS09VKvtSKvsB+7YvB0dEdaFGJRMW+0pSYzFvMfZ0pbIf4H3pqUplX0plP6B79sWXoczMrCCHhZmZFeSweMv1WRfQRUplP8D70lOVyr6Uyn5AN+yL71mYmVlBPrMwM7OCHBYJSddKWixpoaT7JNVkXVNnSfqWpKeT/fmlpGFZ19RZkv5a0hJJbZJ63ZMrkqZLekbScklXZV3PvpB0k6RXJT2VdS37QtI4SQ9IWpr82frbrGvqLEn9JT0uaVGyL19L7bd8GSpH0tCI2Jh8vhKYFBGfzbisTpF0GvC7ZLTCbwBExD9kXFanSDoSaAN+CHwpInrNCFeSyoFngQ8DK8mN/HheRCzNtLBOknQKsAm4JSKOyrqezpI0GhgdEU9IGgIsAP6qN/53UW4YyEERsUlSJfAH4G8j4tGu/i2fWSR2BkViEL14PPeIuC8iWpLZR4GxWdazLyJiWUQ8k3UdnTQNWB4RKyJiB3A70JBxTZ0WEQ+RG/64V4uINRHxRPL5TWAZMCbbqjoncjYls5XJlMqxy2GRR9K/SHoZOB/431nX00UuA36ddRF91Bjg5bz5lfTSg1KpkjQeeDfwWLaVdJ6kckkLgVeBeRGRyr70qbCQdL+kpzqYGgAi4qsRMQ74KXBFttXuWaF9Sdp8FWghtz89VjH7YtbVJA0Gfg58vt2VhV4lIlojopbcFYRpklK5RFiRxpf2VBHxoSKb/hS4F7g6xXL2SaF9kXQJ8DHgg9HDb0ztxX+X3mYVMC5vfmyyzDKWXN//OfDTiPhF1vV0hYh4XdIDwHSgyx9C6FNnFnsiaULebAPwdFa17CtJ04G/B+ojYkvW9fRh84EJkg6RVAWcC8zOuKY+L7kpfCOwLCK+k3U9+0JS9c6nHSUNIPcwRSrHLj8NlZD0c+Bd5J68+Qvw2Yjolf8XKGk50A9Ynyx6tBc/2fVx4P8C1cDrwMKI+Ei2VRVP0hnAd4Fy4KaI+JeMS+o0ST8DTiXXw+la4OqIuDHTojpB0snAw8CT5P6+A/zPiLg3u6o6R9IU4GZyf77KgDsj4ppUfsthYWZmhfgylJmZFeSwMDOzghwWZmZWkMPCzMwKcliYmVlBDguzvSBpU+FWe9z+LkmHJp8HS/qhpOclLZD0oKTjJVVJekhSn3pp1no2h4VZN5E0GSiPiBXJopnkOuabEBHHApcCI5NOB38LnJNNpWbv5LAw6wTlfCvpw+pJSecky8sk/SAZT2SepHslfTLZ7HxgVtLuMOB44B8jog0gIl6IiHuStncn7c16BJ/mmnXOJ4BaYCq5N5rnS3oIOAkYD0wCDiDX/fVNyTYnAT9LPk8m9zZ6626+/ynguFQqN+sEn1mYdc7JwM+SHj/XAr8nd3A/Gfh/EdEWEa8AD+RtMxpoKubLkxDZkQzOY5Y5h4VZ99kK9E8+LwGmJqPp7U4/YFvqVZkVwWFh1jkPA+ckA89UA6cAjwOPAGcl9y5Gket4b6dlwOEAEfE80Ah8LekFFUnjJX00+TwCWBcRzd21Q2Z74rAw65xfAouBRcDvgL9PLjv9nNyIeEuBW4EngDeSbe7h7eExAxgFLJf0FPBjcqOdAbw/aW/WI7jXWbMuJmlwRGxKzg4eB06KiFeS8QYeSOZ3d2N753f8ArgqIp7thpLNCvLTUGZd71fJgDRVwLXJGQcRsVXS1eTG4X5pdxsnAyXd7aCwnsRnFmZmVpDvWZiZWUEOCzMzK8hhYWZmBTkszMysIIeFmZkV5LAwM7OC/j+hb1ni1GYx3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 1\n",
    "scores =  np.zeros((n_classes,n_Cs))        \n",
    "scores[0][:] = np.mean(lr_cv_L2.scores_[1],axis = 0)\n",
    "\n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.01, 0.1, 1, 10, 100, 1000],\n",
       "           class_weight=None, cv=5, dual=False, fit_intercept=True,\n",
       "           intercept_scaling=1.0, max_iter=100, multi_class='ovr',\n",
       "           n_jobs=1, penalty='l2', random_state=None, refit=True,\n",
       "           scoring='neg_log_loss', solver='lbfgs', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [0.001,0.01,0.1,1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1.0: array([[-0.59469018, -0.490727  , -0.45449872, -0.45443454, -0.45480734,\n",
       "         -0.45485278, -0.45485614],\n",
       "        [-0.60901068, -0.54716658, -0.53140586, -0.53082923, -0.53086256,\n",
       "         -0.53086747, -0.53087131],\n",
       "        [-0.61497463, -0.56155136, -0.55663751, -0.56570344, -0.56740653,\n",
       "         -0.56759071, -0.56760797],\n",
       "        [-0.60101253, -0.52415989, -0.50481841, -0.50462784, -0.50478596,\n",
       "         -0.50480412, -0.50481048],\n",
       "        [-0.59044272, -0.4823467 , -0.4377005 , -0.43335094, -0.43307006,\n",
       "         -0.43304606, -0.43303892]])}"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.1])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.C_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAHbdJREFUeJzt3X2clXWd//HXe4YZwHvN0RBQtHATFVFHvM2ttgytoNRU0pRKXS0f1ra1abW56a92f1n9avOWyNT9mUp4k674UDTTXGRlcLkR8AZJA1QYIRVSwWE++8e5Bs4Mw5zDzLnmOjfv5+NxHlzXdb7XOZ/rIV5vrs+5bhQRmJmZ9aQu6wLMzKz8OSzMzKwgh4WZmRXksDAzs4IcFmZmVpDDwszMCnJYmJlZQQ4LMzMryGFhZmYFDci6gFLZfffdY8SIEVmXYWZWUebMmfNaRDQVGlc1YTFixAhaWlqyLsPMrKJIeqmYcW5DmZlZQQ4LMzMryGFhZmYFOSzMzKwgh4WZmRXksDAzs4IcFmZmVlDNh8Xrb23g5w89z+JX3sy6FDOzspVqWEgaJ+lZSUskXbKVMadJWiRpoaTf5C0/R9Lzyeuc1GpEXPXI89z1PyvS+gozs4qXWlhIqgeuBk4ERgETJY3qMmYkcClwbEQcCHwtWb4bcBlwJDAWuEzSrmnUufN2DRz3/t25b/4rREQaX2FmVvHSPLIYCyyJiKURsQG4DZjQZcx5wNUR8ReAiFiVLP84MCMi1iTvzQDGpVXoSQcPYcXrbzN/+RtpfYWZWUVLMyyGAsvy5pcny/LtD+wv6b8kzZI0bhvWLZkTRr2Xhnpx34JX0voKM7OKlvUP3AOAkcCHgInALyXtUuzKks6X1CKppbW1tddFuBVlZtazNMNiBTA8b35YsizfcuCeiHg3Iv4EPEcuPIpZl4iYHBHNEdHc1FTwDrs9civKzGzr0gyL2cBISftKagTOAO7pMuZuckcVSNqdXFtqKfAAcIKkXZMftk9IlqXGrSgzs61LLSwiog24iNxOfjEwNSIWSrpc0vhk2APAakmLgEeAb0bE6ohYA1xBLnBmA5cny1LjVpSZ2dapWnaMzc3N0deHH/22ZRnfnDaf333lWA4ZXvRPJ2ZmFUvSnIhoLjQu6x+4y4pbUWZm3XNY5HErysysew6LLnxWlJnZlhwWXbgVZWa2JYdFF25FmZltyWHRDbeizMw6c1h0w60oM7POHBbdcCvKzKwzh8VWuBVlZraZw2Ir3IoyM9vMYbEVbkWZmW3msOiBW1FmZjkOix64FWVmluOw6IFbUWZmOQ6LAtyKMjNzWBTkVpSZmcOiILeizMwcFkX5xOi9WPH628xzK8rMapTDoggfG7UnDfViultRZlajHBZF2HmwW1FmVtscFkVyK8rMapnDokhuRZlZLXNYFMmtKDOrZQ6LbeBWlJnVKofFNnArysxqlcNiG7gVZWa1ymGxjdyKMrNa5LDYRm5FmVktclhsI7eizKwWpRoWksZJelbSEkmXdPP+JEmtkuYmr3Pz3vuRpIWSFkv6d0lKs9Zt4VaUmdWa1MJCUj1wNXAiMAqYKGlUN0Nvj4gxyWtKsu4xwLHAaOAg4Ajgb9OqdVu5FWVmtSbNI4uxwJKIWBoRG4DbgAlFrhvAIKARGAg0ACtTqbIX3Ioys1qTZlgMBZblzS9PlnV1iqT5kqZJGg4QEU8AjwCvJK8HImJxirVuM7eizKyWZP0D973AiIgYDcwAbgKQ9H7gAGAYuYD5iKQPdl1Z0vmSWiS1tLa29mPZbkWZWW1JMyxWAMPz5oclyzaJiNURsT6ZnQIcnkx/BpgVEesiYh1wP3B01y+IiMkR0RwRzU1NTSXfgJ64FWVmtSTNsJgNjJS0r6RG4AzgnvwBkobkzY4HOlpNfwb+VtIASQ3kftwuqzYUuBVlZrUjtbCIiDbgIuABcjv6qRGxUNLlksYnwy5OTo+dB1wMTEqWTwNeABYA84B5EXFvWrX2lltRZlYrVC0tlObm5mhpaen37/3Cr5/kuZXrePxbH6aMLgUxMyuKpDkR0VxoXNY/cFc8t6LMrBY4LPrIrSgzqwUOiz7yWVFmVgscFiXgVpSZVTuHRQm4FWVm1c5hUQI7D27ggyOb3Ioys6rlsCiRkw4e4laUmVUth0WJuBVlZtXMYVEibkWZWTVzWJSQW1FmVq0cFiXkVpSZVSuHRQm5FWVm1cphUWJuRZlZNXJYlJhbUWZWjRwWJeZWlJlVI4dFCtyKMrNq47BIgVtRZlZtHBYpcCvKzKqNwyIlbkWZWTVxWKTErSgzqyYOi5S4FWVm1cRhkSK3osysWjgsUuRWlJlVC4dFityKMrNq4bBImVtRZlYNHBYpcyvKzKpBUWEh6VhJ2yfTZ0n6qaR90i2tOrgVZWbVoNgji2uBtyQdAvwj8AJwc2pVVRm3osys0hUbFm2R+2fxBOCqiLga2DG9sqqLW1FmVumKDYu1ki4FzgLuk1QHNBRaSdI4Sc9KWiLpkm7enySpVdLc5HVu3nt7S3pQ0mJJiySNKLLWsuNWlJlVumLD4nRgPfCliHgVGAZc2dMKkuqBq4ETgVHAREmjuhl6e0SMSV5T8pbfDFwZEQcAY4FVRdZaltyKMrNKVvSRBfDziPijpP2BMcCtBdYZCyyJiKURsQG4jVwbq6AkVAZExAyAiFgXEW8VWWtZ6mhF3Tf/5axLMTPbZsWGxWPAQElDgQeBzwM3FlhnKLAsb355sqyrUyTNlzRN0vBk2f7A65LulPQ/kq5MjlQ6kXS+pBZJLa2trUVuSjY6WlHTF7zqVpSZVZxiw0LJv+xPBq6JiM8CB5Xg++8FRkTEaGAGcFOyfADwQeAbwBHAfsCkritHxOSIaI6I5qamphKUky63osysUhUdFpKOBs4E7ity3RXA8Lz5YcmyTSJidUSsT2anAIcn08uBuUkLqw24GzisyFrLlltRZlapig2LrwGXAndFxEJJ+wGPFFhnNjBS0r6SGoEzgHvyB0gakjc7Hlict+4ukjoOFz4CLCqy1rLlVpSZVaoBxQyKiEeBRyXtIGmHiFgKXFxgnTZJFwEPAPXADUnQXA60RMQ9wMWSxgNtwBqSVlNEbJT0DeBhSQLmAL/s3SaWl5MOHsLvn1nFvOVvMGb4LlmXY2ZWlKLCQtLB5E5l3S03q1bg7IhY2NN6ETEdmN5l2ffypi8ld8TS3bozgNHF1FdJ8ltRDgszqxTFtqGuB74eEftExN7kbvlRFf/S729uRZlZJSo2LLaPiE2/UUTEH4DtU6moBvisKDOrNMWGxVJJ/yxpRPL6LrA0zcKqmc+KMrNKU2xYfBFoAu5MXk3JMusFt6LMrNIUezbUXyhw9pNtG58VZWaVpMewkHQvsNV/+kbE+JJXVCN8VpSZVZJCRxY/7pcqalB+K+rbJx1A7nISM7Py1GNYJBfjWUrcijKzSlHsRXkL2LId9QbQAvyfiFhd6sJqgVtRZlYpij0b6n5yNxA8M3ndSy4oXqXwrcptK3xWlJlViqKOLICPRkT+XV8XSHoqIg6TdFYahdUKt6LMrBIUe2RRL2lsx4ykI8jdHBByNwG0XvIFemZWCYoNi3OBX0n6k6QXgV8B50raHvjXtIqrBW5FmVklKPaivNnAwZJ2Tubzb2o0NY3CaolbUWZW7oo6spC0s6SfAg+Te8bETzqCw/rOrSgzK3fFtqFuANYCpyWvN4Ffp1VUrXEryszKXbFh8b6IuCx5JvbSiPg+sF+ahdWaT/i25WZWxooNi7clHdcxI+lY4O10SqpNH3UryszKWLFhcSFwtaQXJb0EXAVckF5ZtcetKDMrZ0WFRUTMjYhDyD0T++CIODQi5qVbWu3paEXNXfZ61qWYmXVS6BblX9/KcgAi4qcp1FSzOlpR0xe8wqF775p1OWZmmxQ6stixwMtKyK0oMytXhW5R/v3+KsRyPpFcoDd32es+ujCzslHsD9ybSHoqjUIsJ78VZWZWLrY5LAA/0i1FbkWZWTnqTVjcV/IqrBOfFWVm5WabwyIivptGIbaZW1FmVm6KvZHgWklvdnktk3SXJN/2o8TcijKzclPskcXPgG8CQ4FhwDeA3wC3kbvJYLckjZP0rKQlki7p5v1JklolzU1e53Z5fydJyyVdVewGVQu3osysnBQbFuMj4vqIWBsRb0bEZODjEXE70O35nZLqgauBE4FRwERJo7oZentEjEleU7q8dwXwWJE1VhW3osysnBQbFm9JOk1SXfI6DXgneW9rfZKxwJLkLrUbyB2FTCi2MEmHA3sCDxa7TjVxK8rMykmxYXEm8HlgFbAymT5L0mDgoq2sMxRYlje/PFnW1SmS5kuaJmk4gKQ64Cfk2l01y60oMysXxd5IcGlEfCoido+IpmR6SUS8HRGP9+H77wVGRMRoYAZwU7L8y8D0iFje08qSzpfUIqmltbW1D2WUJ7eizKxcFHs21P6SHpb0dDI/WlKhU2hXAMPz5oclyzaJiNURsT6ZnQIcnkwfDVwk6UXgx8DZkv6t6xdExOSIaI6I5qampmI2paK4FWVm5aLYNtQvgUuBdwEiYj5wRoF1ZgMjJe0rqTEZf0/+AElD8mbHA4uTzz8zIvaOiBHkWlE3R8QWZ1PVAreizKwcFBsW20XEk12WtfW0QkS0kfs94wFyITA1IhZKulzS+GTYxZIWSpoHXAxMKr702uBWlJmVgx7vOpvnNUnvIznzSdKpQMG9V0RMB6Z3Wfa9vOlLyR2x9PQZNwI3Flln1clvRX37pAM2PUvEzKw/FXtk8RXgeuADklYAX8OPVe03bkWZWdaKDYsVwK+BH5C7XmIGcE5aRVlnbkWZWdaKDYvfAZ8i9wP3y8A64K9pFWWd+awoM8tasb9ZDIuIcalWYj3yE/TMLEvFHlnMlHRwqpVYj9yKMrMsFRsWxwFzkjvIzpe0QNL8NAuzznYe3MDxbkWZWUaKbUOdmGoVVpSTDh7Cw25FmVkGigqLiHgp7UKssPxWlMPCzPpTb57BbRlxK8rMsuKwqDAn+QI9M8uAw6LC+KwoM8uCw6LCuBVlZllwWFQgt6LMrL85LCqQW1Fm1t8cFhXIrSgz628OiwrlVpSZ9SeHRYVyK8rM+pPDokK5FWVm/clhUcHcijKz/uKwqGBuRZlZf3FYVDC3osysvzgsKpxbUWbWHxwWFc6tKDPrDw6LCudWlJn1B4dFFXAryszS5rCoAm5FmVnaHBZVwK0oM0ubw6JKuBVlZmlyWFQJt6LMLE2phoWkcZKelbRE0iXdvD9JUqukucnr3GT5GElPSFooab6k09Ossxq4FWVmaUotLCTVA1cDJwKjgImSRnUz9PaIGJO8piTL3gLOjogDgXHAzyTtklat1WLCoUNZ8frbXHLHAto2tmddjplVkQEpfvZYYElELAWQdBswAVhUaMWIeC5v+mVJq4AmwA35Hnxq9BCWrFzLv/9+Ca+tW89VnzuMwY31WZdlZlUgzTbUUGBZ3vzyZFlXpyStpmmShnd9U9JYoBF4IZ0yq4ckvn7C33DFpw/i98+u4nNTZrHmrxuyLsvMqkDWP3DfC4yIiNHADOCm/DclDQH+A/hCRGzRV5F0vqQWSS2tra39UnAl+PxR+3DtmYez8OU3OfW6mSxb81bWJZlZhUszLFYA+UcKw5Jlm0TE6ohYn8xOAQ7veE/STsB9wHciYlZ3XxARkyOiOSKam5qaSlp8pRt30Hu55dwjeW3tek6+diaLXn4z65LMrIKlGRazgZGS9pXUCJwB3JM/IDly6DAeWJwsbwTuAm6OiGkp1ljVjhixG9MuPIYBdeL0659g5guvZV2SmVWo1MIiItqAi4AHyIXA1IhYKOlySeOTYRcnp8fOAy4GJiXLTwOOByblnVY7Jq1aq9n+e+7IHRcew5BdBjHphtncO+/lrEsyswqkajknv7m5OVpaWrIuo2y98da7nHdzC0++uIbvfXIUXzxu36xLMrMyIGlORDQXGpf1D9zWT3beroGbvzSWjx+4J5f/5yL+9f7FtLdXxz8UzCx9DosaMqihnmvOPJzPH7UP1z+6lH/87Tw2tPniPTMrLM2L8qwM1deJyyccyJ47DeTHDz7Ha+vWc+1Zh7PDQP9VMLOt85FFDZLERR8ZyY9OGc3MF1YzcfIsWteuL7yimdUsh0UNO+2I4fzy7MN5ftVaTrl2Ji++9tesSzKzMuWwqHEf+cCe3HreUax9511OuXYm85f79ltmtiWHhXHo3rsy7cJjGNxYzxmTZ/Hoc751ipl15rAwAN7XtAN3XngM+7xne75042zufGp51iWZWRlxWNgme+w0iKl/fxRj992Nr0+dx3WPvuAHKZkZ4LCwLnYc1MCvv3AEnzpkL/7t/mf4/r2LfPGemfk6C9vSwAH1/Pz0Meyx40B+9fifaF23np+edggDB/hBSma1ymFh3aqrE//8yVG8d6dB/GD6YlavW8/ks5vZaVBD1qWZWQbchrIenXf8fvzs9DG0vPgXTrvuCVa++U7WJZlZBhwWVtCnDx3KDZOOYNmatzj5mpksWbUu65LMrJ85LKwox+/fxO1/fzTr2zZy6nUzmfPSX7Iuycz6kcPCinbQ0J2548Jj2GVwA2dOmcVDi1ZmXZKZ9ROHhW2Tfd6zPdMuPIb999yR8/+jhdue/HPWJZlZP3BY2DbbfYeB3HreUXxwZBOX3LmAnz/0vC/eM6tyDgvrle0HDmDKOc2cfNhQ/t9Dz/Gdu59moy/eM6tavs7Ceq2hvo6ffPYQ9txpENf+4QVa167nFxMPZVCDL94zqzY+srA+kcS3xn2Af/nUKB5avJIzp/w3r7+1IeuyzKzEHBZWEpOO3ZerJh7GguVvcOp1T7Di9bezLsnMSshhYSXzidFDuOmLY1n5xjuccs1Mnnn1zaxLMrMScVhYSR39vvcw9YKjCYLPXvcEs5auzrokMysBh4WV3AFDduKOC49hjx0HcvYNT3L/gleyLsnM+shhYakYtut2TLvgGA7aaye+/JunuGnmi1mXZGZ94LCw1Oy6fSO3nHsUf/eBPbnsnoVc+cAzvnjPrEI5LCxVgxvrue6sw5g4djhXP/IC35w2n3c3tmddlpltI1+UZ6kbUF/HDz9zMHvsOIifP/w8r61bzzVnHsZ2jf7rZ1YpUj2ykDRO0rOSlki6pJv3J0lqlTQ3eZ2b9945kp5PXuekWaelTxL/8LH9+eFnDuax51qZOHkWq9etz7osMytSamEhqR64GjgRGAVMlDSqm6G3R8SY5DUlWXc34DLgSGAscJmkXdOq1frP547cm+vOOpxnXl3Lqdc9wZ9Xv5V1SWZWhDSPLMYCSyJiaURsAG4DJhS57seBGRGxJiL+AswAxqVUp/WzEw58L7eceyRr/rqBk6+dydMr3si6JDMrIM2wGAosy5tfnizr6hRJ8yVNkzR8W9aVdL6kFkktra2tparb+kHziN2448KjaawXp1//BI8//1rWJZlZD7I+G+peYEREjCZ39HDTtqwcEZMjojkimpuamlIp0NLz/j125M4vH8uwXbfjCzc+ye/mrsi6JDPbijRPR1kBDM+bH5Ys2yQi8u8FMQX4Ud66H+qy7h9KXqFl7r07D2LqBUdz3s0tfPW2uax6cz3nHb9f1mVZnohgY3vQ1h60RxAB0eV92Lys06U00fFHbPHe5vFbXz82f0CnMfnjivrsbi7vKcUlP0HfP6QUdTQOqGOvXQb3/YN6kGZYzAZGStqX3M7/DOBz+QMkDYmIjntBjAcWJ9MPAD/M+1H7BODSFGu1DO08uIGbvziWr0+dyw+mL2blm+/w7ZMOoK5OWZfWo4hgfVs7Gza2s/7d3J8b2tpZ37Yx+TM3v6GtnXc3tnfa4bZt3LwD3hjBxo3tuemOMR3vbbFOew+f0bFO92M6z7fTHtDW3r5pvfZI1t+4+bs3JkFh5W3M8F24+yvHpvodqYVFRLRJuojcjr8euCEiFkq6HGiJiHuAiyWNB9qANcCkZN01kq4gFzgAl0fEmrRqtewNaqjnFxMPY/cdFjLl8T+xau16rvzsaAYO6PwgpYjg3Y2R7KA35u2gN//Z3c46N72x0469u3U6f9aWn99puh8uLqyvE/US9XViQJ2or+9pvi43vi7v/ToxsKGOweqYr9u0PH9M5/k66uvoNHZAnahLpjsyXGwOc3XJdeUtUJcx6mbcFu/1uP6W39t1TKdxW4zZ8rP7ouu2Z/EZu27X2PciClC13H6hubk5Wlpasi7D+igiuOYPL3DlA8/StONAGuvrOu/MN7aX5LBdgsb6OgYOqKNxQD0DB3RMb/4zN12fG9dQR2N93rIu4wcO6Pxe13Ua6usY0HXH3uOOP/enSrEnMuuBpDkR0VxonC+htbIiia98+P3s857teGDhShrqxcBkZ75pB91pR1y/1Z38wO52/Mn6DfXeEZttC4eFlaVPjt6LT47eK+syzCyR9amzZmZWARwWZmZWkMPCzMwKcliYmVlBDgszMyvIYWFmZgU5LMzMrCCHhZmZFVQ1t/uQ1Aq81IeP2B2ohocqVMt2gLelXFXLtlTLdkDftmWfiCj4jIeqCYu+ktRSzP1Ryl21bAd4W8pVtWxLtWwH9M+2uA1lZmYFOSzMzKwgh8Vmk7MuoESqZTvA21KuqmVbqmU7oB+2xb9ZmJlZQT6yMDOzghwWCUlXSJovaa6kByVV7MMUJF0p6Zlke+6StEvWNfWWpM9KWiipXVLFnbkiaZykZyUtkXRJ1vX0haQbJK2S9HTWtfSFpOGSHpG0KPm79dWsa+otSYMkPSlpXrIt30/tu9yGypG0U0S8mUxfDIyKiAsyLqtXJJ0A/D55Dvr/BYiIb2VcVq9IOgBoB64HvhERFfPsXEn1wHPAx4Dl5J4pPzEiFmVaWC9JOh5YB9wcEQdlXU9vSRoCDImIpyTtCMwBPl2J/12Ue9zj9hGxTlID8Djw1YiYVerv8pFFoiMoEtsDFZuiEfFgRLQls7OAYVnW0xcRsTgins26jl4aCyyJiKURsQG4DZiQcU29FhGPAWuyrqOvIuKViHgqmV4LLAaGZltV70TOumS2IXmlsu9yWOSR9ANJy4Azge9lXU+JfBG4P+siatRQYFne/HIqdKdUrSSNAA4F/jvbSnpPUr2kucAqYEZEpLItNRUWkh6S9HQ3rwkAEfGdiBgO3AJclG21PSu0LcmY7wBt5LanbBWzLWalJmkH4A7ga106CxUlIjZGxBhyHYSxklJpEQ5I40PLVUR8tMihtwDTgctSLKdPCm2LpEnAJ4G/izL/YWob/rtUmhXA8Lz5Yckyy1jS378DuCUi7sy6nlKIiNclPQKMA0p+EkJNHVn0RNLIvNkJwDNZ1dJXksYB/wSMj4i3sq6nhs0GRkraV1IjcAZwT8Y11bzkR+FfAYsj4qdZ19MXkpo6znaUNJjcyRSp7Lt8NlRC0h3A35A78+Yl4IKIqMh/BUpaAgwEVieLZlXwmV2fAX4BNAGvA3Mj4uPZVlU8SScBPwPqgRsi4gcZl9Rrkm4FPkTuDqcrgcsi4leZFtULko4D/ggsIPf/O8C3I2J6dlX1jqTRwE3k/n7VAVMj4vJUvsthYWZmhbgNZWZmBTkszMysIIeFmZkV5LAwM7OCHBZmZlaQw8JsG0haV3hUj+tPk7RfMr2DpOslvSBpjqQ/SDpSUqOkxyTV1EWzVt4cFmb9RNKBQH1ELE0WTSF3Y76REXE48AVg9+Smgw8Dp2dTqdmWHBZmvaCcK5N7WC2QdHqyvE7SNcnzRGZImi7p1GS1M4HfJePeBxwJfDci2gEi4k8RcV8y9u5kvFlZ8GGuWe+cDIwBDiF3RfNsSY8BxwIjgFHAHuRuf31Dss6xwK3J9IHkrkbfuJXPfxo4IpXKzXrBRxZmvXMccGtyx8+VwKPkdu7HAb+NiPaIeBV4JG+dIUBrMR+ehMiG5OE8ZplzWJj1n7eBQcn0QuCQ5Gl6WzMQeCf1qsyK4LAw650/AqcnD55pAo4HngT+Czgl+e1iT3I33uuwGHg/QES8ALQA30/ugoqkEZI+kUy/B3gtIt7trw0y64nDwqx37gLmA/OA3wP/lLSd7iD3RLxFwP8HngLeSNa5j87hcS6wJ7BE0tPAjeSedgbw4WS8WVnwXWfNSkzSDhGxLjk6eBI4NiJeTZ438Egyv7Uftjs+407gkoh4rh9KNivIZ0OZld5/Jg+kaQSuSI44iIi3JV1G7jncf97aysmDku52UFg58ZGFmZkV5N8szMysIIeFmZkV5LAwM7OCHBZmZlaQw8LMzApyWJiZWUH/C2+W/Txt9YueAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 1\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "scores[0][:] = np.mean(lrcv_L2.scores_[1],axis = 0)\n",
    "\n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4970121977345695\n"
     ]
    }
   ],
   "source": [
    "print(min(mse_mean))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic 回归参数选择\n",
    "通过以上分析，选择正则函数是L1，超参数C取值为1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM\n",
    "## 线性SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LinearSVC不能得到每类的概率，在Otto数据集要求输出每类的概率，这里只是示意SVM的使用方法\n",
    "#https://xacecask2.gitbooks.io/scikit-learn-user-guide-chinese-version/content/sec1.4.html\n",
    "#1.4.1.2. 得分与概率\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn import metrics\n",
    "\n",
    "SVC1 = LinearSVC().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.84      0.89      0.86       100\n",
      "        1.0       0.77      0.69      0.73        54\n",
      "\n",
      "avg / total       0.82      0.82      0.82       154\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[89 11]\n",
      " [17 37]]\n"
     ]
    }
   ],
   "source": [
    "#在校验集上测试，估计模型性能\n",
    "y_predict = SVC1.predict(X_test)\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (SVC1, metrics.classification_report(y_test, y_predict)))\n",
    "print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 线性SVM正则参数调优\n",
    "线性SVM LinearSVC的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1）\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_grid_point_Linear(C, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC2 =  LinearSVC( C = C)\n",
    "    SVC2 = SVC2.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC2.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.8181818181818182\n",
      "accuracy: 0.8181818181818182\n",
      "accuracy: 0.8181818181818182\n",
      "accuracy: 0.8181818181818182\n",
      "accuracy: 0.8116883116883117\n",
      "accuracy: 0.7597402597402597\n",
      "accuracy: 0.5194805194805194\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAH6xJREFUeJzt3XuUFOWd//H3h+ugYiAwJioo6CIRL/EyokdINKIIXkAqv0RMzFHX6G4SL9nNxrjJnpjo7ibunhNzWZONyWbNZhOJGxHwioh4jSiDd0AUMeoQo4h4IYLcvr8/qojtMNDNzFRX98zndU6f7qqu6v7WAfpDPU/V8ygiMDMz254eRRdgZma1z2FhZmZlOSzMzKwsh4WZmZXlsDAzs7IcFmZmVpbDwqyTSfqDpOOLrsOsMzkszHIi6UBJsyW9Jsk3NFldc1iY5WcDcD1wbtGFmHVUr6ILMOuqImIpsFTSXxVdi1lH+czCzMzKcliYmVlZDgszMyvLYWFmZmW5g9ssJ5IE9AX6ZMsNQETEu4UWZtYOPrMwy8/ewFpgUba8FlhaXDlm7SdPfmRmZuX4zMLMzMpyWJiZWVkOCzMzK8thYWZmZXWZS2cHDx4cw4YNK7oMM7O6snDhwtciorHcdl0mLIYNG0Zzc3PRZZiZ1RVJL1SynZuhzMysLIeFmZmV5bAwM7OyukyfhZlZd7dhwwZaWlpYt27dVu81NDQwZMgQevfu3a7PdliYmXURLS0t9O/fn2HDhpGOY5mKCFatWkVLSwvDhw9v12e7GcrMrItYt24dgwYNel9QAEhi0KBBbZ5xVMphYWbWhbQOinLrK9Xtm6H+/Ge48sqiq7Curn9/aGyE3XZ779HYCP36FV2ZWWW6fVi88w788z8XXYV1ZdubBWCXXd4LjtIQaev14MHQp0/16jYr1e3DorERNm8uugrryiJgzRpYuRJefTV9tPX6xRehuTld3rix7c8aMKB8qGx5PWgQ9OxZ3WO14kVEm01OHZ27qNuHhVnepLQZqn9/2Gef8ttHwBtvbDtUtjyeeQbuvx9ee63tsxcpPRup5KylsREGDkz3sfrV0NDAqlWrturk3nI1VENDQ7s/22FhVmOk9Id74EAYObL89ps2weuvbztUtiw/9lj6evXqtj+nV680NNrqW9nyeo894LDDoIcvjalJQ4YMoaWlhZUrV2713pb7LNrLYWFW53r2fO9H/oADym+/fn16NrKtUNnyevny9PWaNe/ff9w4+MUvYK+98jkea7/evXu3+z6KchwWZt1Mnz7pGcIee1S2/dq17wXJgw/C178OBx4I3/8+nHOOm666C59Mmtl29euXnkU0NcGFF8KTT8Lhh8O558Ipp8Af/1h0hVYNDgsz2yHDhsHcufCDH8C8eelZxm9+s/1LhK3+OSzMbIf16AEXXZR2mo8cCZ/9LHzqU2lzlXVNuYaFpAmSlkpaJunSNt7fS9I8SY9KekLSSSXv/WO231JJJ+ZZp5m1z377pZfvXnkl3HRT2sE+fXrRVVkecgsLST2Bq4GJwCjgDEmjWm32T8D1EXEoMBX4cbbvqGz5AGAC8OPs88ysxvTsCZdcAgsXwtCh8MlPwplnbvsSXatPeZ5ZjAaWRcTyiFgPTAMmt9omgF2z1x8AtnSVTQamRcS7EfE8sCz7PDOrUQceCPPnw7e+Bb/9bbp8221FV2WdJc+w2BN4qWS5JVtX6lvAmZJagFuBC3dgXySdL6lZUnNbN6GYWXX17g2XXQYPPZTeVHjSSXDeefDWW0VXZh1VdAf3GcC1ETEEOAn4laSKa4qIayKiKSKaGhsbcyvSzHbMYYelzVKXXprewHfQQXDXXUVXZR2RZ1isAIaWLA/J1pU6F7geICIeBBqAwRXua2Y1rG9f+M534IEHoKEhvfP7wgvTaQGs/uQZFguAEZKGS+pD2mE9q9U2LwLjACTtTxoWK7PtpkrqK2k4MAJ4OMdazSwnRx0Fjz4KF18M//EfcMghaYBYfcktLCJiI3ABMBtYQnrV0yJJl0ualG32FeA8SY8D1wFnR2oR6RnHYuB24EsRsSmvWs0sXzvtlA4PMm9eOvz6xz6WXkHVgVk+rcrU0THOa0VTU1M0NzcXXYaZlfH22/DVr8JPfwqjRsEvf5kOJWLFkLQwIsr+CRTdwW1m3Uz//vCf/wm33w5vvpk2U33zm+louFa7HBZmVogTT4SnnkqHCrniCjjySHjiiaKrsm1xWJhZYQYMSJuhZsxIR69taoJ//ddtTytrxXFYmFnhJk+GRYtgyhT4xjdgzBh4+umiq7JSDgszqwmDB6fDhEybBsuWwaGHwlVXwebNRVdm4LAwsxpz+unpWcYJJ8Df/z0ceyw891zRVZnDwsxqzoc/DDNnwrXXwuOPw0c/Cj/5iSdYKpLDwsxqkgRnnZVeMXX00fDFL8L48fDii0VX1j05LMyspg0dCrNnp/dmPPhgOijhtdf6LKPaHBZmVvMk+Ju/Se/DOOQQOOccmDQJXn656Mq6D4eFmdWNffZJx5e66iq48850gqVp03yWUQ0OCzOrKz16wJe/DI89BiNGwBlnwKc/DZ7/LF8OCzOrSyNHwv33p3NmzJyZnmXMmFF0VV2Xw8LM6lavXulsfAsXwh57pHeAf+5zsHp10ZV1PQ4LM6t7Bx2Uzvv9zW/CddelZxm33150VV2Lw8LMuoQ+feDb34b589MBCidOhPPPT+fPsI5zWJhZl9LUlDZLXXIJ/PzncPDB6RVU1jEOCzPrchoa4Mor4b770n6N446Diy6Cd94purL65bAwsy5rzJj0EtsLL4Qf/Si9oe/3vy+6qvrksDCzLm3nneGHP4S5c9OpWz/2Mfja12DduqIrqy8OCzPrFo47Lh0u5Nxz4d/+LR2c0PN+V85hYWbdxq67wjXXwM9+Bo8+6o7vHeGwMLNu58wz0+apG28supL64bAws26noQFOOikdHmTTpqKrqQ+5hoWkCZKWSlom6dI23r9K0mPZ4xlJb5S8t6nkvVl51mlm3U+SwCuvpHNkWHm98vpgST2Bq4ETgBZggaRZEbF4yzYR8Xcl218IHFryEWsj4pC86jOz7u2kk9K7vm+8EcaOLbqa2pfnmcVoYFlELI+I9cA0YPJ2tj8DuC7HeszM/mLXXeH442H6dM+HUYk8w2JP4KWS5ZZs3VYk7Q0MB+4qWd0gqVnSfEmnbWO/87Ntmld6MHsz20FJAn/4Q3rjnm1frXRwTwV+FxGlXU17R0QT8Bng+5L2bb1TRFwTEU0R0dTY2FitWs2si5g0KZ1MyVdFlZdnWKwAhpYsD8nWtWUqrZqgImJF9rwcuJv392eYmXVYY2N6R/f06UVXUvvyDIsFwAhJwyX1IQ2Era5qkvQRYCDwYMm6gZL6Zq8HA2OAxa33NTPrqCSBRYvgmWeKrqS25RYWEbERuACYDSwBro+IRZIulzSpZNOpwLSI93Ux7Q80S3ocmAd8t/QqKjOzzjJlSvrspqjtU3SRywCampqiubm56DLMrA4dcUTad/HQQ0VXUn2SFmb9w9tVKx3cZmaFSRJ4+GFoaSm6ktrlsDCzbi9J0ucZM4qto5Y5LMys2xs5Evbf31dFbY/DwsyM9OzinnvgtdeKrqQ2OSzMzEjDYvNmuOmmoiupTQ4LMzPg0ENh773dFLUtDgszM0BK77m44w54++2iq6k9Dgszs0ySpPNy33Zb0ZXUHoeFmVnm6KNht93cFNUWh4WZWaZnT5g8GW65BdatK7qa2uKwMDMrkSSwZg3MnVt0JbXFYWFmVuK449JZ9NwU9X4OCzOzEn36wCmnwMyZsHFj0dXUDoeFmVkrSQKrVsF99xVdSe1wWJiZtTJhAjQ0eI6LUg4LM7NWdt4ZTjwx7bfYvLnoamqDw8LMrA1JAitWgOdUSzkszMzacMop0KuXm6K2cFiYmbXhgx+EY4+FG26ALjL7dIc4LMzMtiFJ4NlnYfHioispnsPCzGwbTjstHY3WTVEOCzOzbdp9dzjqKN/NDQ4LM7PtShJ49FF4/vmiKymWw8LMbDumTEmfZ8woto6i5RoWkiZIWippmaRL23j/KkmPZY9nJL1R8t5Zkp7NHmflWaeZ2bbsuy8cfLCbonILC0k9gauBicAo4AxJo0q3iYi/i4hDIuIQ4EfA9GzfDwKXAUcCo4HLJA3Mq1Yzs+1JEnjgAXjllaIrKU6eZxajgWURsTwi1gPTgMnb2f4M4Lrs9YnAnIh4PSJWA3OACTnWama2TUmS3msxc2bRlRQnz7DYE3ipZLklW7cVSXsDw4G7dmRfSedLapbUvHLlyk4p2systQMPTJujunNTVEVhIWm6pJMl5RUuU4HfRcSmHdkpIq6JiKaIaGpsbMypNDPr7qT07GLuXHjjjfLbd0WV/vj/GPgM8Kyk70oaWcE+K4ChJctDsnVtmcp7TVA7uq+ZWe6SJJ0M6ZZbiq6kGBWFRUTcGRGfBQ4D/gDcKen3ks6R1Hsbuy0ARkgaLqkPaSDMar2RpI8AA4EHS1bPBsZLGph1bI/P1pmZFWL06PQmve7aFFVxs5KkQcDZwOeBR4EfkIbHnLa2j4iNwAWkP/JLgOsjYpGkyyVNKtl0KjAt4r2huiLideAK0sBZAFyerTMzK0SPHuk9F7fdBu+8U3Q11aeoYDhFSTcCI4FfAddGxMsl7zVHRFN+JVamqakpmj3wvJnlaO5cOP74dKyo004ruprOIWlhJb/hlZ5Z/DAiRkXEd0qDAqAWgsLMrBo+/vF06PLu2BRVaViMkjRgy0LWl/DFnGoyM6tJvXvDqafCTTfBhg1FV1NdlYbFeRHxlwvGshvlzsunJDOz2pUk6eWzd99ddCXVVWlY9JSkLQvZUB598inJzKx2nXAC7Lxz92uKqjQsbgd+K2mcpHGk90Tcnl9ZZma1qV8/mDgxHYV28+aiq6meSsPia8A84AvZYy5wSV5FmZnVsiSBP/0J5s8vupLq6VXJRhGxGfhJ9jAz69ZOPhn69Emboo4+uuhqqqPSsaFGSPqdpMWSlm955F2cmVkt2nVXGDcuDYsKblXrEipthvpv0rOKjcAngP8B/jevoszMal2SpFOtPvFE0ZVUR6Vh0S8i5pLe8f1CRHwLODm/sszMatukSekQIN3lqqhKw+LdbHjyZyVdIGkKsEuOdZmZ1bTddoOxYx0WrV0M7ARcBBwOnAl4Xmwz69aSBJ56Cp59tuhK8lc2LLIb8E6PiDUR0RIR50TEJyOiG100Zma2tSlT0ucbbyy2jmooGxbZ7HVjq1CLmVld2WsvOPzw7tEUVWkz1KOSZkn6nKRkyyPXyszM6kCSwEMPwYouPpdnpWHRAKwCjgNOzR6n5FWUmVm9SLL/Ns+YUWwdeav0Du5z8i7EzKwefeQj6WP6dPjSl4quJj8VhYWk/wa2uk8xIv660ysyM6szSQJXXgmrVsGgQUVXk49Km6FuBm7JHnOBXYE1eRVlZlZPkgQ2bUonReqqKm2GuqF0WdJ1wP25VGRmVmcOOyy9Mmr6dDj77KKryUelZxatjQB268xCzMzqlZTec3HHHbCmi7a5VDrq7NuS3tryAG4inePCzMxIm6LefRduu63oSvJRaTNU/7wLMTOrZ2PGQGNj2hT1qU8VXU3nq/TMYoqkD5QsD5B0Wn5lmZnVl549YfJkuOWW9Ayjq6m0z+KyiHhzy0JEvAFcVm4nSRMkLZW0TNKl29jm09mkSosk/aZk/SZJj2WPWRXWaWZWmCSBt9+GuXOLrqTzVdQMRduhst19swEIrwZOAFqABZJmRcTikm1GAP8IjImI1ZJKO83XRsQhFdZnZla4445LZ9GbPh1OOqnoajpXpWcWzZK+J2nf7PE9YGGZfUYDyyJieUSsB6YBk1ttcx5wdUSsBoiIV3ekeDOzWtK3bzo/98yZ6X0XXUmlYXEhsB74LemP/jqg3I3tewIvlSy3ZOtK7QfsJ+kBSfMlTSh5r0FSc7be/SNmVheSBF57De7vYneiVXo11J+BNvscOuH7RwDHAkOAeyUdlPWJ7B0RKyTtA9wl6cmIeK50Z0nnA+cD7LXXXjmUZ2a2YyZMgIaGtCnqmGOKrqbzVHo11BxJA0qWB0qaXWa3FcDQkuUh2bpSLcCsiNgQEc8Dz5CGBxGxInteDtwNHNr6CyLimohoioimxsbGSg7FzCxXu+wC48enEyLFViPq1a9Km6EGZ//bByDrYyh3B/cCYISk4ZL6AFOB1lc1zSA9q0DSYNJmqeVZGPUtWT8GWIyZWR1IEnjpJVhYrme3jlQaFpsl/aWdR9Iw2hiFtlREbAQuAGYDS4DrI2KRpMslTco2mw2skrQYmAd8NSJWAfuTdqo/nq3/bulVVGZmtezUU9P7LrrSDHqKCs6Tso7na4B7AAEfA86PiHJNUVXT1NQUzc3NRZdhZgbA8cens+ctWVJ0JdsnaWFENJXbrqIzi4i4HWgClgLXAV8B1naoQjOzLixJ4Omnaz8sKlVpB/fnSeex+ArwD8CvgG/lV5aZWX07Lbvgv6s0RVXaZ3ExcATwQkR8gvTKpDe2v4uZWfe1xx5w1FHdLyzWRcQ6AEl9I+JpYGR+ZZmZ1b8kgUcegRdeKLqSjqs0LFqy+yxmAHMkzQS6wOGbmeVnypT0+cYbi62jM1R0NdT7dpCOAT4A3J6N+VQTfDWUmdWigw+GAQPg3nuLrqRtnXo1VKmIuCciZtVSUJiZ1aokSceJeuWVoivpmPbOwW1mZhVIknTYj1l1PiuPw8LMLEcHHQT77FP/V0U5LMzMciSlZxdz58Kbb5bfvlY5LMzMcpYksGFDOj93vXJYmJnl7MgjYffd67spymFhZpazHj3S4T9uuw3W1umoeg4LM7MqSBJ45x24446iK2kfh4WZWRUccwwMHFi/TVEOCzOzKujdO50U6aab0s7ueuOwMDOrkiSB1avhnnuKrmTHOSzMzKpk/HjYaaf6bIpyWJiZVUm/fjBxIsyYAZs3F13NjnFYmJlVUZLAyy/DQw8VXcmOcViYmVXRySennd311hTlsDAzq6IPfADGjUsnRNrB6YQK5bAwM6uyJIHnnoMnnyy6kso5LMzMqmzy5HQ02npqiso1LCRNkLRU0jJJl25jm09LWixpkaTflKw/S9Kz2eOsPOs0M6um3XaDsWPra27u3MJCUk/gamAiMAo4Q9KoVtuMAP4RGBMRBwBfztZ/ELgMOBIYDVwmaWBetZqZVVuSwBNPwLJlRVdSmTzPLEYDyyJieTZf9zRgcqttzgOujojVABHxarb+RGBORLyevTcHmJBjrWZmVTVlSvpcL2cXeYbFnsBLJcst2bpS+wH7SXpA0nxJE3ZgXzOzurX33nDYYQ6LSvUCRgDHAmcAP5M0oNKdJZ0vqVlS88qVK3Mq0cwsH0kCDz4If/xj0ZWUl2dYrACGliwPydaVagFmRcSGiHgeeIY0PCrZl4i4JiKaIqKpsbGxU4s3M8tbkqTPM2YUW0cl8gyLBcAIScMl9QGmArNabTOD9KwCSYNJm6WWA7OB8ZIGZh3b47N1ZmZdxv77w8iR9dEUlVtYRMRG4ALSH/klwPURsUjS5ZImZZvNBlZJWgzMA74aEasi4nXgCtLAWQBcnq0zM+tSkgTmzYPXa/wXTlFP95tvR1NTUzQ3NxddhpnZDmluhiOOgGuvhbMKuKNM0sKIaCq3XdEd3GZm3drhh8PQobXfFOWwMDMrkJTeczF7NqxZU3Q12+awMDMrWJLAunVw++1FV7JtDgszs4KNHQuDB9d2U5TDwsysYD17piPR3nwzvPtu0dW0zWFhZlYDkgTeegvuuqvoStrmsDAzqwHjxkH//rXbFOWwMDOrAX37pvNzz5gBmzYVXc3WHBZmZjUiSWDlSnjggaIr2ZrDwsysRkycmJ5h1GJTlMPCzKxG7LILjB+fzs1dayMxOSzMzGpIksCLL8IjjxRdyfs5LMzMasipp6b3XdRaU5TDwsyshgwaBMcckzZF1RKHhZlZjUkSWLIEnn666Ere47AwM6sxp52WPtdSU5TDwsysxuy5Jxx5ZG01RTkszMxqUJKks+i9+GLRlaQcFmZmNWjKlPR5xoxi69jCYWFmVoNGjIADD6ydpiiHhZlZjUoSuO++dLyoojkszMxqVJLA5s0wa1bRlTgszMxq1sEHw/DhtdEU5bAwM6tRUnp2ceed6Sx6RXJYmJnVsCSB9evh1luLrSPXsJA0QdJSScskXdrG+2dLWinpsezx+ZL3NpWsr4EWOzOz6jvqKPjwh4tviuqV1wdL6glcDZwAtAALJM2KiMWtNv1tRFzQxkesjYhD8qrPzKwe9OiRDv/xq1/B2rXQr19BdeT42aOBZRGxPCLWA9OAyTl+n5lZl5Qk8Oc/w5w5xdWQZ1jsCbxUstySrWvtk5KekPQ7SUNL1jdIapY0X9JpbX2BpPOzbZpX1sKFyGZmOTj2WBgwoNiBBYvu4L4JGBYRBwNzgF+WvLd3RDQBnwG+L2nf1jtHxDUR0RQRTY2NjdWp2Mysynr3TidFmjULNmwopoY8w2IFUHqmMCRb9xcRsSoi3s0Wfw4cXvLeiux5OXA3cGiOtZqZ1bQkgddfh3vvLeb78wyLBcAIScMl9QGmAu+7qknS7iWLk4Al2fqBkvpmrwcDY4DWHeNmZt3G+PFp53ZRTVG5hUVEbAQuAGaThsD1EbFI0uWSJmWbXSRpkaTHgYuAs7P1+wPN2fp5wHfbuIrKzKzb2GknmDgxDYvNm6v//YqI6n9rDpqamqK5ubnoMszMcvPrX8OZZ8KDD6b3X3QGSQuz/uHtKrqD28zMKnTyydCrVzFNUQ4LM7M6MWAAjBuX3s1d7UYhh4WZWR1JEli2DJ56qrrf67AwM6sjkyeno9FWuynKYWFmVkc+9CEYM6b6Aws6LMzM6kySwOOPw/Ll1ftOh4WZWZ2ZMiV9rmZTlMPCzKzODBsGhx5a3aYoh4WZWR1KEvj97+Hll6vzfQ4LM7M6lCTp88yZ1fk+h4WZWR3af3/Yb7/qNUU5LMzM6pCUnl3MmwerV+f/fQ4LM7M6lSSwcSPcfHP+3+WwMDOrU01NMGRIdZqieuX/FWZmlgcJvvAFeOed/L/LYWFmVse+/vXqfI+boczMrCyHhZmZleWwMDOzshwWZmZWlsPCzMzKcliYmVlZDgszMyvLYWFmZmUpIoquoVNIWgm80IGPGAy81knlFKmrHAf4WGpVVzmWrnIc0LFj2TsiGstt1GXCoqMkNUdEU9F1dFRXOQ7wsdSqrnIsXeU4oDrH4mYoMzMry2FhZmZlOSzec03RBXSSrnIc4GOpVV3lWLrKcUAVjsV9FmZmVpbPLMzMrCyHhZmZleWwyEi6QtITkh6TdIekPYquqb0k/bukp7PjuVHSgKJrai9Jn5K0SNJmSXV3maOkCZKWSlom6dKi6+kISb+Q9Kqkp4qupSMkDZU0T9Li7O/WxUXX1F6SGiQ9LOnx7Fi+ndt3uc8iJWnXiHgre30RMCoi/rbgstpF0njgrojYKOlKgIj4WsFltYuk/YHNwE+Bf4iI5oJLqpiknsAzwAlAC7AAOCMiFhdaWDtJ+jiwBvifiDiw6HraS9LuwO4R8Yik/sBC4LR6/HORJGDniFgjqTdwP3BxRMzv7O/ymUVmS1BkdgbqNkUj4o6I2JgtzgeGFFlPR0TEkohYWnQd7TQaWBYRyyNiPTANmFxwTe0WEfcCrxddR0dFxMsR8Uj2+m1gCbBnsVW1T6TWZIu9s0cuv10OixKS/kXSS8BngW8WXU8n+WvgtqKL6Kb2BF4qWW6hTn+UuipJw4BDgYeKraT9JPWU9BjwKjAnInI5lm4VFpLulPRUG4/JABHxjYgYCvwauKDYarev3LFk23wD2Eh6PDWrkmMx62ySdgFuAL7cqmWhrkTEpog4hLQFYbSkXJoIe+XxobUqIo6vcNNfA7cCl+VYToeUOxZJZwOnAOOixjumduDPpd6sAIaWLA/J1lnBsvb9G4BfR8T0ouvpDBHxhqR5wASg0y9C6FZnFtsjaUTJ4mTg6aJq6ShJE4BLgEkR8U7R9XRjC4ARkoZL6gNMBWYVXFO3l3UK/xewJCK+V3Q9HSGpccvVjpL6kV5Mkctvl6+Gyki6ARhJeuXNC8DfRkRd/i9Q0jKgL7AqWzW/jq/smgL8CGgE3gAei4gTi62qcpJOAr4P9AR+ERH/UnBJ7SbpOuBY0uGwXwEui4j/KrSodpA0FrgPeJL03zvA1yPi1uKqah9JBwO/JP371QO4PiIuz+W7HBZmZlaOm6HMzKwsh4WZmZXlsDAzs7IcFmZmVpbDwszMynJYmO0ASWvKb7Xd/X8naZ/s9S6SfirpOUkLJd0t6UhJfSTdK6lb3TRrtc1hYVYlkg4AekbE8mzVz0kH5hsREYcD5wCDs0EH5wKnF1Op2dYcFmbtoNS/Z2NYPSnp9Gx9D0k/zuYTmSPpVkn/L9vts8DMbLt9gSOBf4qIzQAR8XxE3JJtOyPb3qwm+DTXrH0S4BDgo6R3NC+QdC8wBhgGjAJ2Ix3++hfZPmOA67LXB5Dejb5pG5//FHBELpWbtYPPLMzaZyxwXTbi5yvAPaQ/7mOB/4uIzRHxJ2BeyT67Aysr+fAsRNZnk/OYFc5hYVY9a4GG7PUi4KPZbHrb0hdYl3tVZhVwWJi1z33A6dnEM43Ax4GHgQeAT2Z9Fx8iHXhviyXAXwFExHNAM/DtbBRUJA2TdHL2ehDwWkRsqNYBmW2Pw8KsfW4EngAeB+4CLsmanW4gnRFvMfC/wCPAm9k+t/D+8Pg88CFgmaSngGtJZzsD+ES2vVlN8KizZp1M0i4RsSY7O3gYGBMRf8rmG5iXLW+rY3vLZ0wHLo2IZ6pQsllZvhrKrPPdnE1I0we4IjvjICLWSrqMdB7uF7e1czZR0gwHhdUSn1mYmVlZ7rMwM7OyHBZmZlaWw8LMzMpyWJiZWVkOCzMzK+v/A2+cYdCYPaqUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-3, 3, 7)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份  \n",
    "#penalty_s = ['l1','l2']\n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    #for penalty in penalty_s:\n",
    "        tmp = fit_grid_point_Linear(oneC, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "#for j, penalty in enumerate(penalty_s):\n",
    "#for penalty in accuracy_s.keys:\n",
    "pyplot.plot(x_axis, np.array(accuracy_s), 'b-')\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuracy' )\n",
    "#pyplot.savefig('SVM_Otto.png' )\n",
    "pyplot.title(penalty)\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 最优参数\n",
    "线性SVM最优参数是L2正则，超参数C取值为1。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RBF核SVM正则参数调优\n",
    "RBF核是SVM最常用的核函数。 RBF核SVM 的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和核函数的宽度gamma C越小，决策边界越平滑； gamma越小，决策边界越平滑。\n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_grid_point_RBF(C, gamma, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集是那个利用SVC训练\n",
    "    SVC3 =  SVC( C = C, kernel='rbf', gamma = gamma)\n",
    "    SVC3 = SVC3.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC3.score(X_val, y_val)\n",
    "    \n",
    "    print(\"accuracy: {}\".format(accuracy))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.7792207792207793\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.8051948051948052\n",
      "accuracy: 0.7857142857142857\n",
      "accuracy: 0.6753246753246753\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.8051948051948052\n",
      "accuracy: 0.7662337662337663\n",
      "accuracy: 0.6818181818181818\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.8246753246753247\n",
      "accuracy: 0.7142857142857143\n",
      "accuracy: 0.6818181818181818\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 2, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-2, 2, 5)  \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.7792207792207793\n",
      "accuracy: 0.6493506493506493\n",
      "accuracy: 0.6818181818181818\n",
      "accuracy: 0.8051948051948052\n",
      "accuracy: 0.7857142857142857\n",
      "accuracy: 0.6883116883116883\n",
      "accuracy: 0.8051948051948052\n",
      "accuracy: 0.8051948051948052\n",
      "accuracy: 0.7662337662337663\n",
      "accuracy: 0.8116883116883117\n",
      "accuracy: 0.8051948051948052\n",
      "accuracy: 0.8246753246753247\n",
      "accuracy: 0.7142857142857143\n"
     ]
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "C_s = np.logspace(-2, 2,5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份 \n",
    "gamma_s = np.logspace(-4, -1, 4)  \n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "    for j, gamma in enumerate(gamma_s):\n",
    "        tmp = fit_grid_point_RBF(oneC, gamma, X_train, y_train, X_test, y_test)\n",
    "        accuracy_s.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzs3Xd8U9X7wPHPSboH3ZRRoOxSVtkyyhQFnAjIdKK4QBTFvQdftyIoTkSlDGUoKiDIaplSBWS0QCkFWkYX3Svj/P5I6K9ACwWa3rQ579crL5qbe2+ehDZPzj3nPEdIKVEURVGUq6XTOgBFURSlZlOJRFEURbkmKpEoiqIo10QlEkVRFOWaqESiKIqiXBOVSBRFUZRrohKJoiiKck1UIlEURVGuiUokiqIoyjVx0jqA6hAYGChDQ0O1DkNRFKVG+eeff9KllEGX288hEkloaCixsbFah6EoilKjCCGOVWY/dWlLURRFuSYqkSiKoijXRCUSRVEU5ZqoRKIoiqJcE5smEiHEECHEQSFEghDiuXIebyyE2CCE2CWE+E8IMcy6fbAQ4h8hxF7rvwPLHLPRes7d1ltdW74GRVEU5dJsNmpLCKEHPgMGA8nATiHECinlgTK7vQT8JKWcI4QIB1YCoUA6cIuU8qQQoh3wJ9CwzHHjpZRqGJaiKIodsGWLpDuQIKVMlFKWAIuA2y7YRwJ1rD/7ACcBpJS7pJQnrdv3A+5CCFcbxqooiqJcJVsmkobAiTL3kzm/VQHwGjBBCJGMpTUypZzzjAD+lVIWl9n2nfWy1stCCFGFMSuKotR4xaZitqRs4b2d72EwGWz+fFpPSBwLzJNSfiiE6An8KIRoJ6U0Awgh2gLvAjeUOWa8lDJFCOENLAXuAn648MRCiEnAJIDGjRvb+GUoiqJoKyUvhc3Jm4lJiWHHqR0UmYpw1btya/NbCfMPs+lz2zKRpACNytwPsW4rayIwBEBKuU0I4QYEAqlCiBBgOXC3lPLIuQOklCnWf3OFEAuwXEK7KJFIKb8CvgLo2rWrrKoXpSiKYg8MJgP/pv5LTHIMMSkxJGYnAtDQqyHDWw4nsmEk3ep1w83Jzeax2DKR7ARaCiGaYkkgY4BxF+xzHBgEzBNCtAHcgDQhhC/wB/CclHLLuZ2FEE6Ar5QyXQjhDNwM/GXD16AoimI3zuSfYXOKpdWx7eQ2CowFOOuc6RLchREtRxAZEklonVCq+4q/zRKJlNIohJiMZcSVHpgrpdwvhHgDiJVSrgCeAr4WQjyJpeP9XimltB7XAnhFCPGK9ZQ3APnAn9YkoseSRL621WtQFEXRktFs5L+0/4hJiSEmOYaDZw8CUM+zHjc1u4nIhpH0qN8DD2cPTeMUUtb+qz5du3aVqmijoig1QXphOltSthCTEsPWk1vJLclFL/R0qtuJyJBIIhtG0sK3RbW0OoQQ/0gpu15uP6072xVFURyayWxif8b+0lbH/oz9AAS6B3J94+vp07APPRv0xNvFW+NIK6YSiaIoSjXLKspi68mtxKTEsCVlC2eLz6ITOjoEdmBKpylENoyktX9rdKJmVLFSiURRFMXGzNJMfGZ86Qirvel7MUszfq5+9G7Ym8iGkfRq0AtfN1+tQ70qKpEoiqLYQG5JLttObiMmJYbNKZtJL0wHoF1AOx7q8BCRDSMJDwhHr9NrHOm1U4lEURSlCkgpOZx1mJhkS+LYlboLkzTh7eJN7wa9iQyxtDoC3QO1DrXKqUSiKIpylQoMBWw/tb20o/xMwRkAWvu15r529xHZMJIOQR1w0tXuj9ra/eoURVGqkJSSozlHS0uR/HPmHwxmA57OnvSs35NHQx6ld4PeBHsGax1qtVKJRFEU5RIKjYXsPL2ztKM8Jc9S6am5T3PGtxlPZMNIOtXthLPeWeNItaMSiaIoygVO5J4oTRw7T++k2FSMu5M7Per14L6299EnpA8NvS4sZm5/SoxmXJxsP4RYJRJFURxeiamE2DOxpR3lSTlJADSp04RRrUYR2TCSLvW64KqvOcsi7UjM4InFu5l7bzfa1K9z+QOugUokiqI4pFN5pyyd5Nay64XGQlx0LnSr140xYWPo07APTeo00TrMqyKlZMaqeKSE0ABPmz+fSiSKojgEg9nA7tTdpSOsErISAGjg2YBbm99K35C+dA3uqnkBxKqwcu9p9pzI4r2RHXB3sf08FZVIFEWptdIK0s4ru55nyMNJONEluAu3d72dyIaRNPVpWu1l122pxGjmvT/jaR3szYjOIdXynCqRKIpSa5jMJvam7yU6OZrNKZuJy4wDoK57XW4MvbG07LqXi5fGkdrOgh3HOJZRwHf3dUOvq54EqRKJoig1WmZR5nll17OLs9ELPR2DOjK181QiG0bSyq9VrWp1VCSnyMCn6xPo1TyA/q2Cqu15VSJRFKVGMUszBzIOlA7P3Ze+D4nE382ffiH9iAyJpGf9nvi4+mgdarX7ctMRMvNLeH5om2pNnCqRKIpGErMT+eXwL0hq/+JyVaIkn/SUv9ladIpMcwkCaO/sxyNerenrWpc2zr7oCgQc2my5OZi8YiP+O0/wbQNP2h/YBgesD/SaCl62bZ2oRKIoGnn373fZfmp7jZqboCljEe4mI9cVG4gsNNCruAR/czpwWOvI7IKL0cxYIXHL0cHOMq2RzveoRKIotVFiViJbT25lcsRkHur4kNbh2L+j0fD9LXD969DnCa2jsTvxp3MYNjOG+3s35aWbw6v9+W06d14IMUQIcVAIkSCEeK6cxxsLITYIIXYJIf4TQgwr89jz1uMOCiFurOw5FaUmiIqLwkXnwqjWo7QOxf6ZzbD2FagTAj1U0i3Pu6vi8XJ1YvLAFpo8v80SiRBCD3wGDAXCgbFCiAtT5UvAT1LKTsAY4HPrseHW+22BIcDnQgh9Jc+pKHYtuzib3xJ/Y1izYfi7+Wsdjv3bvwxO7oKBL4Gzu9bR2J2tCelsOJjGYwNa4OvhokkMtmyRdAcSpJSJUsoSYBFw2wX7SOBcERgf4KT159uARVLKYinlUSDBer7KnFNR7Nqyw8soNBYyoc0ErUOxf8ZiWPcGBLeHDndqHY3dMZslM1bF0dDXnXt6hWoWhy0TSUPgRJn7ydZtZb0GTBBCJAMrgSmXObYy51QUu2U0G1kYv5CuwV1p7d9a63Ds385vIesYDH4dasGStFXtt/9Osi8lh6duaIWbs3bvj+3rC1/aWGCelDIEGAb8KISokpiEEJOEELFCiNi0tLSqOKWiXLONJzZyKv+Uao1URmEWRL8HzQZAi0FaR2N3io0m3lt9kPD6dbg9Qtvv07ZMJClAozL3Q6zbypoI/AQgpdwGuAGBlzi2MufEer6vpJRdpZRdg4Kqb4anolzK/Lj5NPBsQP9G/bUOxf5t/tiSTAa/rnUkdunHbcdIySrkhWFt0FVTKZSK2DKR7ARaCiGaCiFcsHSer7hgn+PAIAAhRBssiSTNut8YIYSrEKIp0BL4u5LnVBS7FJ8Zzz9n/mFs2Fj06jLNpWWdgO1zoMNoqN9R62jsTnaBgVnrE+jbKog+LQO1Dsd280iklEYhxGTgT0APzJVS7hdCvAHESilXAE8BXwshnsTS8X6vlFIC+4UQP2GZm2kEHpNSmgDKO6etXoOiVKX5B+bj7uTO8JbDtQ7F/m2YYfl34IvaxmGnPt+YQE6RgeeGhGkdCmDjCYlSypVYOtHLbnulzM8HgN4VHPs28HZlzqko9i6jMIOVR1dyR8s7HLIG1BU5vRf2LIReU8C3sdbR2J3kswV8tzWJOzqFEN7AtisfVpbWne2K4hB+PvQzBrOBcW3GaR2K/Vv7Krj5QOQ0rSOxSx+tOQTAUze00jiS/6cSiaLYmMFkYPHBxfRu0JtmPs20Dse+HVkPR9ZB3+ng7qd1NHZnX0o2y3encH/vpjTwtZ/JmSqRKIqNrTm2hvTCdMa3Ga91KPbtXCkU38bQ/UGto7FL766Ox8fdmUf6N9c6lPOoRKIoNhYVF0VonVB6Nyy3O1A5Z+/Plv6Rga+Ak6qIfKHoQ2nEHE5nysCW+Lg7ax3OeVQiURQb2pO2h73pexkbNhZd1cy1rZ0MRbD+TctQ33YjtI7G7pjMkv+tiqeRvzsTrrO/AQjqN1tRbCjqQBRezl7c1kKVhLukv7+C7BMw+E3QqY+lC/2yK4W4UzlMvzEMVyf7m4Ok/scUxUbO5J9h7bG1DG85HE9nT63DsV8FmRDzAbQYDM36aR2N3SkymPhwzUE6hPhwc/v6WodTLpVIFMVGFh9cjEmaGBs2VutQ7FvMh1CUo0qhVGDe1iROZhfx3NAwzUuhVEQlEkWxgSJjET8f+pn+jfrTyLvR5Q9wVGePWS5rRYyH4LZaR2N3zuaX8NmGBAaG1aVXc+1LoVREJRJFsYGVR1eSVZylqvxezvq3QOhgwAtaR2KXZm9IIL/YyLN2UgqlImrNdkWpYlJKouKiaOHbgm71ugFgLirCmJ6OMTUNY3oaxjTLDSHw7NkTj06dEM72NaTT5k7ugr0/QZ9p4KOWFbrQicwCftiWxKgujWhdz1vrcC5JJRJFuUpSSszZ2aVJwZiejjEtjeRj+xiyL45O+lASv7wZY1oa5tzci0+gt4y+yZjzBTovLzx798arb188I/vgXLduNb+aaiYlrHkZ3P2hzxNaR2OX3v/zIHqd4MnB9lMKpSIqkSjKBaTBgDEjw5og0sskCut9a4vClJaONBguOl646GjlKfBv6otLq2A8e/XCKSjIegu0/BsYiN7PD3NhIfnbtpEfHU3epmhy//wTALfwcDz79cWrb1/cO3RA6O1vyOc1SfgLkmJgyLuWulrKef5LzmLFnpNMHtCCej5uWodzWcJStb1269q1q4yNjdU6DEVjprx8jGmpmNLTz29FpJ7fojCdPVvu8Xo/v/9PCIGBONUt83NQEPrAQFLdDdz050gmtp/I1M5Tryg+KSXFBw+StymavOhoCnftArMZvY8PnpGRePWNxLNPH5z8/avi7dCO2QRf9AFDATy2E5xctI7IrkgpGfv1dg6dyWPT9P54u2l3yVMI8Y+Usuvl9lMtEqVGk2YzpszM0iRg6YNIv+hykzE9HVlQcNHxwtkZvbWV4NyoEe6dO1mTwwUtiICASvVhLNr5PjqhY3Tr0Vf8WoQQuIWF4RYWRuBDkzBlZ5O/daslscTEkPP77yAEbh3a49W3L159++HWNhxR0ybw7VkIqQdg5HcqiZRj48E0tidm8vqtbTVNIldCtUgUu2QuLv7/ZJB2fnIwlb3clJkJJtNFx+u8vc9rLVx4WencNp2PD0JUzdj8AkMB1/98Pb0b9ub9fu9XyTnPkWYzRfsPkBe9ibzoaIr+2wtSog8IwCsyEq9+ffHs1Qu9j51fJiopgFldwLsePLgequi9ry1MZsnQmdGUGM2sebIfLk7afklQLRLF7pR2TpdtMaSlX5QojOnpmHNyLj6BTodTQEBpC8I1vM35iSIwyHK5KTAQnVv1X1f+9civ5BpybVLlV+h0uLdvh3v7dgQ99hjGzEzyN2+2tFY2bCD7l19Ar8c9IsLSWunXF9fWrassSVaZHXMg9ySM+FolkXIs/SeZQ2fy+Hx8Z82TyJVQLRKlypiLisjftq3iVkR6OrKk5KLjhJtbmVZD2RZD4Hnb9P7+dtvpbJZmbvvlNjydPVl408Jq/QCXJhOF//1HXnQ0+ZuiKTpwAACn4GBLv0rfvnj27IXeS+MyLfnpMDMCQvvAuEXaxmKHCktM9P9gA/V93Fn+aC+7+BKgWiRKtUueOpX8TdGl9/W+vqWJwDU0tPRn/blEYW1B6Dw97eKP5lpsPbmVpJwkZvSZUe2vRej1eHTqhEenTjB1KobUVPJjNpMXHU3OqtVk/bwEnJ3x6NLF2rcSiUvz5tX/nke/D4Z8VQqlAnO3HOVMTjGzx3WucX8PKpEoVaIgNpb8TdEEPPwQfqNHWzqnXRynI3V+3HwC3QMZEjpE61BwrlsX3xF34DviDqTBQMGuXaXDi1Pfe4/U997DuUGD0uHFnj16oPPwsG1QGUdg5zfQ+W4Iam3b56qB0vOKmbPxCDeEB9MttOaNyrNpIhFCDAFmAnrgGynlOxc8/jEwwHrXA6grpfQVQgwAPi6zaxgwRkr5ixBiHtAPyLY+dq+UcrcNX4ZyGVJKUj/5BH1QIIEPPYTO3X6WAK0OidmJbEnZwqMRj+Kst69RNsLZGc/u3fHs3p26Tz+N4dQp8qJjyIuOJvvXFWQtXIRwccGje/fSvhWXJk2qPpD1b4LeBfo/X/XnrgVmrTtMocHEM3ZeCqUiNkskQgg98BkwGEgGdgohVkgpD5zbR0r5ZJn9pwCdrNs3ABHW7f5AArCmzOmnSymX2Cp25crkb95CYew/BL/8ksMlEYAFcQtw1jkzqtUorUO5LOf69fEbfSd+o+/EXFJCYWxs6byVMzNmcGbGDFyaNLG2Vvrh0a0rOtdrXK0wORb2L4d+z1pGaynnOZqeT9SO44zp1ogWdb20Dueq2LJF0h1IkFImAgghFgG3AQcq2H8s8Go520cCq6SUF08CUDQnpSRt5kycGzTAb5T9f5BWtZySHFYcWcHQpkMJdLff6qzl0bm44NmrF569ehH8/HOUnDhBXrQlqWQt/omzP/yIcHfHs0cPvKyXwZwbXmFNLCkt67B7BkGvKbZ5ITXc+3/G4+KkY+r1LbUO5arZMpE0BE6UuZ8M9ChvRyFEE6ApsL6ch8cAH12w7W0hxCvAOuA5KWVxOeecBEwCaNzY/pamrC1y//qLon37qP/22w7VJ3LO8sPLKTQW2mTIb3VzadQI//Hj8R8/HnNREQV//21prWzaRN7GjZZ9WjTHq28/vPr2xaNzp8v/nx9aDce2wE0fgqt9Fx7Uwr/Hz7Jy72meuL4ldb3tvxRKRWw2/FcIMRIYIqV8wHr/LqCHlHJyOfs+C4RIKadcsL0+8B/QQEppKLPtNOACfAUckVK+calY1PBf25AmE0dvvx1pNNHstxUIJ8cau2Eym7hp+U0EewTz/dDvtQ7HZqSUlBxNIi96E/nR0eTvjAWDAZ2np6VF0zfS0loJDj7/QJMR5vQCaYJHt4Od9R9pTUrJnV9u42h6AZum98fT1f7+fuxh+G8KUHZFnxDrtvKMAR4rZ/udwPJzSQRASnnK+mOxEOI74OkqiFW5Cjl//EHx4QQafvShwyURgI3JG0nJS2Fal2lah2JTQghcmzXFtVlTAu69F3N+Pvnbt5f2reSuXQuAa1hYaYe9e8eOiD3zIf0gjJ6vkkg51h44w86ks7w9vJ1dJpErYcvodwIthRBNsSSQMcC4C3cSQoQBfsC2cs4xFnj+gv3rSylPCctA69uBfVUduHJ50mAgbdZsXMPC8B6i/ZBXLUTFRVHfsz4DGw/UOpRqpfP0xHvQILwHDbIUmjx8uHR4cca335Lx1VfovL3xCsrGs3U7vIJ6qnkGFzCazLyzOp5mQZ6M7lrzV9C02f+vlNIohJgM/Ill+O9cKeV+IcQbQKyUcoV11zHAInnBNTYhRCiWFs2mC04dJYQIAgSwG3jYVq9BqVjWsuUYTpwg5PPPa17RwCpwMPMgO0/v5MkuT+Kkc9yPSSEEbq1a4daqFQEPPIApJ4f8rdvI++kz8v7NIicxE1ZF4ta+felkSLd27ey2QkF1WRx7gsS0fL66qwtO+pr/96NKpChXzFxczJEbh+AUXJfQRYtq3CzcqvDKlldYdXQVf436Cx9XOy+UWN3yUuHTTsim/Snq8AL5MTHkbYqmcM8eS1l8Pz88I/vg1bcfnr174eTnp3XE1Sq/2Ei/9zfSNNCDnx7qadd/P/bQR6LUUlmLFmE8fZoG/6v+ciD2ILMokz8S/+C2FrepJFKeTe+CoRAx+HXcA1vg3rYtgQ8/jPHsWfK3bLV02sdsJmfFb6DT4d6xY+nwYtc2bWr979TXMYmk5xXz1d1das1rVYlEuSLm/HzSv/wKjx498OzZU+twNLHk0BJKzCW1YshvlUs/DLHfQdf7ILDFeQ85+fnhc/NN+Nx8E9JkomjfvtIO+7RPZpL2yUycgoKso8D64dmrJ3rv2jVkODW3iK+iExnWvh6dG9eelphKJMoVyfxxPqbMTIKeuLLV/2oLg9nA4vjF9Kzfk+a+zbUOx/6sex2c3S2z2C9B6PW4d+yIe8eOBD0+BWN6Onkxm8mPiSZ37V9kL10GTk54dOpkWWulb19cW7as8d/gZ/51mBKjmek31sxSKBVRiUSpNFNODhlz5+LVv7+l0qwD+uvYX6QWpvJKz1e0DsX+HN8Bcb/BgBfBq+4VHeoUGIjv8NvxHX470mikcM+e0tZK6gcfwgcf4lS/Pt6Dryfo8cfRe9W8UiIJqXks2nmCCT0a0zRQ45L+VUwlEqXSMubOxZyTQ9DUx7UORTPz4+bT2LsxkSGRWodiX6SEtS+DVzD0LG9KWOUJJyc8unTBo0sX6k57EsOZM5bSLZs2cTZqAfmbomn46ae4tW5VRcFXj/dWx+PurGfKoJpbCqUiNX/cmVItjBkZZP7wI95Dh+DWpo3W4Whib9pe/kv7j3FtxqET6k/nPPG/w4kdMOAFcKnab9vOwcH4jRpFo9mzaTLvO0wF+SSNHk32r79W6fPY0s6kTNYcOMPD/ZoR6HWNRTDtkPprUCol46uvkUVFBE1x3MJ78+Pm4+nsyW3Nb9M6FPtiMsBfr0Fga4iYYNOn8ujWjWbLluHevj0nn32OU6++hrn4olJ7dkVKyYyVcQTXcWVin2Zah2MTKpEol2U4fZqzCxfic9ttuDarnX8Il5NakMqapDUMbzEcL5ead33epv79HjISLCsf6m1/tdwpKIjG380l4MEHyFq8mGPjxlOSXFH1Je2t3neaXcezmDa4Fe4utXMipkokymWlz/kCKSWBj13bte+abPHBxZikiXFhF1X5cWzFubDxHWjSG1pVX6kc4eRE3aeeIuSz2ZQcP87RESPItVYoticGk5l3V8fTKtiLEZ1DtA7HZlQiUS6p5MQJspYuxW/USFxCrnAtilqi2FTMkkNL6BfSj0Z1an5dpCq1dRbkp8HgN0GDobnegwbRdOkSnOvXJ/nhR0j95BOkyVTtcVRk4d/HScoo4LmhYbWiFEpFau8rU6pE+uzZCL2egIcct6TZqqOryCzKZFwb1Ro5T+5pSyJpOxxCumgWhkvjxoQuWojPiDvI+OJLjj/wAMaMDM3iOSe3yMDMvw5zXTN/BrS+suHQNY1KJEqFihMSyF7xG37jx+McXLv/ECoipSQqLooWvi24rv51WodjXzb+z9LRPkj7OTU6NzcavP029d9+i8J/d3H0jhEU/LtL05i+ik4kI7+E54fW/rIvKpEoFUr7dBY6Dw8CHnxA61A088+Zf4jPjGdcm3G1/sPgiqQdhH9/gG4Twd9+BmD4jhhB6KKFCBcXjt19N5nff48WhWnP5BTxdUwit3RsQMdGvtX+/NVNJRKlXIX795O7Zg3+99zjcNVZy4qKi8LH1Yebm92sdSj2Ze2r4OIFfadrHclF3Nq0oenSJXj168eZ/71DypPTMOXlVWsMH689hMksmX5D62p9Xq2oRKKUK23mTHQ+Pvjfd6/WoWgmJS+F9SfWM6LlCNyd3LUOx34kbYZDq6DPE+AZqHU05dLXqUPI7FnUffopctesIWnkKIoOHaqW5z50JpefYk9wd89QGgd4VMtzak0lEuUiBf/+S350DAEPTKx11VevxKL4RQgEY8PGah2K/ZAS1rwM3g2gxyNaR3NJQggCHniAxvO+w5SXR9LoMWSvWHH5A6/Ru6vi8XR1YvKAFpffuZZQiUQ5j5SStI8/QR8YiP94xy2TXmAoYOnhpQxqPIh6nvW0Dsd+7F8OJ/+FgS+CS834tu3ZvTtNly3FrW04J595llOvv465pMQmz7XtSAbr4lN5bEAL/DxdbPIc9qhSiUQIsUwIcZMQqsBQbZe/dSsFO3cS+NBD6DxqxgeFLfye+Du5JblqzZGyjCWWMvF120LHmtVKc65blybz5uE/8X6yFi7i2PgJGFKqdja82Sz536o4Gvi4cW+v0Co9t72rbGL4HBgHHBZCvCOEcIweJAcjpbQsLtSgPr6j79Q6HM2cG/Lbxr8Nneo6Zrn8csXOhbNJMPgN0NW8Uh/CyYng6dMJmT2LkqNHOXrHCPKio6vs/L/vPcV/ydk8dUNr3Jxr3vtzLSqVSKSUf0kpxwOdgSTgLyHEViHEfUII54qOE0IMEUIcFEIkCCGeK+fxj4UQu623Q0KIrDKPmco8tqLM9qZCiB3Wcy4WQjhO+9HG8tavp2jvXoIefRSdi+O+rdtObiMxO5EJ4RPUkN9zirItS+g27QctBmkdzTXxvv56mi5dglO9epx46GHSPv30mmfDFxtNvP9nPG3q1+H2To5XAaLSl6qEEAHAvcADwC5gJpbEsraC/fXAZ8BQIBwYK4QIL7uPlPJJKWWElDICmAUsK/Nw4bnHpJS3ltn+LvCxlLIFcBaYWNnXoFRMms2kzfwUlyZN8Ln9dq3D0dT8uPkEuAUwJLT6akfZvc2fQGGmpTVSC5KrS5MmhC5ehM/w4aR/PocTD07CmJl51eebv/04JzILeX5oGHpdzX9/rlRl+0iWAzGAB3CLlPJWKeViKeUUoKJSqN2BBCllopSyBFgEXKr+9lhg4WXiEMBAYIl10/eAY3/qVZGclasoPnSIwClTEE6Ou95ZUnYSMSkx3Nn6Tlz0jtsqO092Cmz/HNrfCQ0itI6myujc3Ggw423qv/UmBbGxltnwu658Nnx2oYFZ6w8T2TKQvq2CbBCp/atsi+RTKWW4lPJ/UspTZR+QUnat4JiGwIky95Ot2y4ihGgCNAXWl9nsJoSIFUJsF0KcSxYBQJaU0liJc06yHh+blpZ2yRfn6KTRSPqsWbi2akWdYUO1DkdTC+IX4KRz4s7WjttHdJENM0CaYeBLWkdiE74jR9Jk4QKEkxPH7rqbzB9+vKLZ8HM2HiG70MCzQ2rXOuxXorKJJFwIUTrPXwjhJ4R4tArjGAMskVKWvVDZxJqkxgGfCCGaX8kJpZRfSSl8FmP7AAAgAElEQVS7Sim7BgU55reEysr+5RdKjh0jaOrjCJ3jDszLLcnl14RfGRo6lEB3+5xoV+3O7IfdUdB9Evg10Toam3Fv29YyGz4ykjMzZpAybRqmvPzLHpeSVcjcLUcZHtGQdg19qiFS+1TZT40HpZSlHeFSyrPAg5c5JgUoW3M7xLqtPGO44LKWlDLF+m8isBHoBGQAvkKIc9deLnVOpRLMJSWkffY5bh064DVwoNbhaOqXhF8oMBaoIb9lrX0V3OpA5FNaR2Jzeh8fQj6bTdC0aeT+uYakO++k+PDhSx7z0RrLbPlpN9Ss9eOrWmUTiV6UGb5i7Ui/3AXknUBL6ygrFyzJ4qJppUKIMMAP2FZmm58QwtX6cyDQGzggLe3NDcBI6673ADVn4WY7lLVoMcZTpyytkVrQiXq1TGYTC+IWEBEUQdvAtlqHYx8SN0LCWoh8Gjz8tY6mWgidjsBJD9J47lxM2dkcvXM02b/9Xu6+B07msGxXMvf1CiXEz3HnXEHlE8lqYLEQYpAQYhCW1sPqSx1g7ceYDPwJxAE/SSn3CyHeEEKUHYU1Blgkz78o2QaIFULswZI43pFSHrA+9iwwTQiRgKXP5NtKvgblAuaCAtK//BKPbt3w7NVL63A0FZ0cTXJeMuPDVWsEALMZ1r4CPo0sl7UcjOd1PWi6bBlu4eGcnD6d02+8cdFs+HdWx1PHzZlH+ztOKZSKVHZ4zrPAQ8C54jprgW8ud5CUciWw8oJtr1xw/7VyjtsKtK/gnIlYRoQp1yhzfhSmjAyCZn3q0K0RsFT5DfYIZlDjmj1HosrsWwqn9sDwr8DZTetoNOEcXJcm874j9aOPyfzuOwr37Sfk449wbtiQmMNpRB9K46Wb2uDjUeFUOodRqUQipTQDc6w3pRYw5eSQ8e23ePaNxKNzZ63D0dShs4fYcXoHUztPxVmnPhQwFsO6N6Bee2g/SutoNCWcnQl+9hncO0Vw6vkXOHrHCOq/9x7/2y0I8XPnrp61dwDClajsPJKWQoglQogDQojEczdbB6fYTua8eZizswmaOlXrUDS3IG4Bbno3RrYcefmdHcHfX0P2ccs67A48iq+sOjfcYJkNHxzMiYcfpvP6n5k+uAWuTo5VCqUilf0t+Q5La8QIDAB+AObbKijFtoyZmWTO+x7vG27Ava1jdyxnFWXxe+Lv3NTsJnzdav9KdpdVeBai34fmg6D5AK2jsSsuoaHUmx/FtmbdGX9wLRGfvYHx7Fmtw7ILlU0k7lLKdYCQUh6z9mvcZLuwFFvK+PobzEVFBD0+RetQNLfk8BKKTcWMazNO61DsQ8xHlrpag1/XOhK7NH/3Gd5sN5L8ydMp3LmTo3eMoHDPHq3D0lxlE0mxtYT8YSHEZCHEcCoujaLYMcOZVM4uWIDPLbfg2sKxR5sYzAYWxS+iR70etPJz7HkAAGQdhx1fWkrE1yt3rItDyyooYfb6BPqH1aXr5PtpsmABQqcjacJdZM6P0mRteHtR2UQyFUudrceBLsAELHM4lBom/Ys5SJOJwMmPaR2K5tYdX8eZgjNqAuI569+2/DvwRW3jsFOfbUggt9jIc0MtpVDc21lnw/fqxZm33uLkU09jzr/8bPja6LKJxDr5cLSUMk9KmSylvE9KOUJKub0a4lOqUElyMlk/L8F35AhcGjW6/AG1XNSBKEK8Qugb0lfrULR3ag/8txiuewR8QrSOxu6cyCzg+63HGNk5hLB6dUq36319CZnzOUFPPknO6tUcvXM0xUeOaBipNi6bSKz1r/pUQyyKjaXP/gyh1xP4iH2vtV0d9qfvZ3fabsa1GYe+Bi7SVOXWvgruvtDnSa0jsUsfrjmIEOWXQhE6HYEPTaLx3G8xnT3L0VF3kv3HHxpEqZ3KXtraJYRYIYS4Swhxx7mbTSNTqlTxkSNkr1iB39ixOAcHax2O5ubHzcfDyYPbW6hVCEhYB4kboO8zlmSinGdfSja/7D7JxD5Nqe/jXuF+ntddR9Ply3ALC+PkU09z+s23kDZaG97eVDaRuGEpmDgQuMV6u9lWQSlVL23WbHRubgRMulytzdovrSCN1Umrub3F7Xi7eGsdjrbMJktrxLcJdFNrxF1ISsmMlXH4e7rwcP/LFyB3Dg6myffz8L/nHs5GRZF0110YTp267HE1XWVntt9n60AU2yk6cIDc1asJeORhnPwdo/jepfx86GeMZiNjw8ZqHYr2/vsJzuyFEd+Ck6vW0didTYfS2Hokg9duCaeOW+WqHghnZ4Kffw73Tp049eKLHB1+Bw0++ACvPr1tHK12KpVIhBDfAReNbZNS3l/lESlVLm3mp+jq1CHgPvV9oMRUwuKDi4lsGEmoT6jW4WjLUAjr34IGnaCtulJ9IZNZ8s6qeJoEeDCux5WXQqkz5EZcW7ci5fGpnHjwQQInP0bgI4/UyjV/KvuKfgf+sN7WAXWAPFsFpVSdgl27yNu0iYCJE9HXqXP5A2q51UmrySzKZEKbCVqHor0dX0JOsmUd9lr44Xatlv2bTPzpXJ65MQwXp6t7f1ybNrWsDX/rLaTPms2JSQ/Vytnwlb20tbTsfSHEQmCzTSJSqlTaJzPRBwTgf5f64JRSMv/AfJr5NKNng55ah6OtgkzLLPaWN0JTNfz5QkUGEx+uOUTHRr4Ma1/vms6l8/Cg/jvv4N6pM2fefpujd4wgZOYnuHfoUEXRau9qv4a0BOpWZSBK1cvfto2CHTsIfGgSOg/HXngHYFfqLuIy4xjfZrzDl80n+gMoyYXrX9M6Ers0d8tRTucU8cLQsCr5XRFC4DdmtGU2vBAkjZ9A5oIFtWY2fGWr/+YKIXLO3YDfsKxRotgpKSWpn3yCU716+I4erXU4dmF+3Hy8Xby5uZmDDzg8mwR/fwUR4yE4XOto7E5mfglzNhzh+jbB9GgWUKXndm/fjqbLluLZqydn3niTk9OfqRWz4SuVSKSU3lLKOmVurS683KXYl7wNGyna8x+Bjz6CzlWNxjmVd4r1x9czsuVIPJwdvHW27k3QOcGAF7SOxC7NWn+Y/BIjzw1tbZPz6319aTRnDkFPTCXnjz84Ono0xYk1e1WOyrZIhgshfMrc9xVCqJlcdkqazaTNnIlz48b4Dh+udTh2YdHBRUgkY8LGaB2KtlL+hX1LoOdjUKeB1tHYnWMZ+czffozR3RrToq7t5hgJnY7Ahx+m8bffYMo8S9LIUeSsWmWz57O1yvaRvCqlzD53R0qZBbxqm5CUa5W7ejXFBw8SNGUywlmt+FdoLGTJoSUMbDSQBl4O/OEpJax5GTwCobda0Kw87/15ECedjievb1ktz+fZqxdNly3FtVUrUp6cxum3Z9TI2fCVTSTl7XfZEV9CiCFCiINCiAQhxHPlPP6xEGK39XZICJFl3R4hhNgmhNgvhPhPCDG6zDHzhBBHyxwXUcnX4BCk0Ujap7NwbdmCOsOGaR2OXfg98XdySnJUld/Da+DYZuj/HLipoeAX2nX8LH/8d4oH+zajbp3qW6feuV49mvzwPX5338XZH3/k2F13Yzh9utqevypUNpHECiE+EkI0t94+Av651AHWqsGfAUOBcGCsEOK8nj0p5ZNSyggpZQQwC1hmfagAuFtK2RYYAnwihChbBGj6ueOklLsr+RocQvavKyhJSiLw8ccRelWMUErJgrgFhPmH0SW4i9bhaMdkhLWvgH9z6HKv1tHYHSkl/1sVT6CXC5P6Nqv25xcuLtR74QUafvIxxYcPc3T4HeRt2VLtcVytyiaSKUAJsBhYBBQBl1vQojuQIKVMlFKWWI+77RL7jwUWAkgpD0kpD1t/PgmkAkGVjNVhmUtKSP/sM9zatcP7+uu1DscubD+1nYSsBDXkd88CSIuH618FvbrceaF1can8fTSTqde3wsu1UtPrbKLOkCGELlmCU2AAJx54kLTPP0eazZrFU1mVHbWVL6V8TkrZVUrZTUr5gpTycmPWGgInytxPtm67iBCiCdAUWF/OY90BF6Bskf+3rZe8PhZCqCFJVlk//4zh5EmCpk517A/NMqLiovB382do06Fah6KdknzYMANCukGbW7WOxu4YTWbeWR1Ps0BPxnTTfp0e12ZNCV28mDo330z6p7M48fDDdj8bvrKjttaWvbQkhPATQvxZhXGMAZZY1z4p+7z1gR+B+6SU59Ly80AY0A3wp4L5LEKISUKIWCFEbFpaWhWGap/MhYWkf/EF7l274FmLi8NdieM5x4lOjmZUq1G46h34+8b2zyH3FAx+E9QXjIv8/E8yCal5PDMkDGe9fZSK0Xl40OC9d6n36isUbNvO0REjKNy7V+uwKlTZdy3QOlILACnlWS4/sz0FKJveQ6zbyjMG62Wtc4QQdbDU9nqx7GqMUspT0qIY+A7LJbSLSCm/sragugYF1f6rYmcXLMCUlk7dJ55QrRGrhfEL0Qs9d7a+U+tQtJOXBptnQtjN0MTBy8KUo6DEyEdrD9GliR83trWvdXqEEPiNHUuTBVEAHBs3nrMLF9rlbPjKJhKzEKLxuTtCiFDKqQZ8gZ1ASyFEUyGEC5ZkseLCnYQQYYAfsK3MNhdgOfCDlHLJBfvXt/4rgNuBfZV8DbWWKS+PjK++xrNPHzy6dtU6HLuQV5LH8oTl3BB6A3U9HLiaT/R7YCiAQWq0fnm+iTlKWm4xLwyrmlIotuDevj1Nly7F47rrOP36G5x85lnMBQVah3WeyiaSF4HNQogfhRDzgU1YLjFVSEppBCYDfwJxwE9Syv1CiDeEEGUv1I4BFsnz0+ydQF/g3nKG+UYJIfYCe4FA4K1KvoZaK/O7eZiyswmaquYGnPPrkV/JN+Q7dpXfjCMQOxe63ANBFy8R6+jScov5ctMRhrStR5cm9r1Oj5OfH42+/ILAx6eQ8/vvJI0eTXHiUa3DKiUq20wSQtQFJgG7AHcgVUoZbcPYqkzXrl1lbGys1mHYhPHsWY5cPxjPXj0JmTVL63DsglmauWX5Lfi6+RI1LErrcLTz091w+C94fBd429dlG3vw8i/7WPD3cdY+2ZdmQV5ah1NpeVu2cPLp6cjiYurPeJs6Q4bY7LmEEP9IKS97maOyne0PYFmH5CngaSwd4K9dS4BK1cj45hvMBQUEPf641qHYjZjkGI7nHnfs1siJnXDgV+j9uEoi5TiSlseCv48zrnvjGpVEALx697bMhm/ZkpQnnuTM//6HNBg0jamyl7amYhkldUxKOQDoBGRd+hDF1gypqZyNWkCdm2/GtWX1lHSoCebHzaeuR12ub+Kgc2mkhLUvg2dd6DlZ62js0vurD+LmpOPxQTXz78a5fn2a/PgDfnfdReb3P3Ds7ns0nQ1f2URSJKUsAhBCuEop4wHblMZUKi3jiy+RRiNBky83N9RxJJxNYPup7YxpPQZnnYNOvDu4Eo5vgwHPg2vN+rZdHf45lsnq/ad5qF9zgrxr7rBw4eJCvRdfoOFHH1J08CBH7xhB/rZtlz/QBiqbSJKt80h+AdYKIX4FjtkuLOVySpJTOPvzz/jecQcuTa58PenaakH8Alx0LoxoNULrULRhMsLaVyGgJXS6W+to7I6Ukhkr4wnyduWByKZah1Ml6gwbRtOff0Lv58fxiQ+Q/sUX1T4bvrIz24dLKbOklK8BLwPfYhl6q2gk/fPPEUIQ+OgjWodiN7KLs/ntyG/c1Owm/N3sexSOzez6ATIOw+DXQa9dqQ979ef+M/xz7CzTBrfCw6X2vD+uzZvT9KfF1Bk6lLRPZnLikUcwZVVf78MVT+OUUm6SUq6w1s9SNFCceJTsX37Bb+wYnOtd23rStcnSw0spMhU5bpXf4jzY8D9o3BNaq8rPFzKYzLy3Op6Wdb0Y1SVE63CqnM7TkwYfvE/wyy+Rv3UbR+8YQeHe6plmZx/1AJQrkj57FsLNjYBJk7QOxW4YzUYWxi+kW71utPZ30O67bbMhP1WVQqnAop0nSEzP57mhYTjZSSmUqiaEwH/8eELn/4iUkmPjxlVLaZXa+W7WYkXx8eSsXIX/XXfhFFC160nXZOuPr+d0/mnHbY3knoEtn0L4bdCom9bR2J28YiMz/zpEj6b+DAyr/ZUO3Dt2pOmypQQ8+ABu4eGXP+Aa1Z6LhA4ibean6Ly9Cbj/Pq1DsStRcVE09GpI/5D+WoeijU3vgKlYlUKpwFfRiaTnlfDNPW3sthRKVXPy86u2+WWqRVKDFO7eTd6GDQRMvB+9j4/W4diNAxkH+Df1X8aGjUWvc8DFvNIOwT/fQ9f7IaC51tHYndScIr6OTuTmDvWJaOR7+QOUK6YSSQ2SOnMmen9//O+6S+tQ7EpUXBTuTu4Mbzlc61C0se51cPaAvs9oHYld+vivwxjNZqbf6KB9Z9VAJZIaIn/7Dgq2bSdg0oPoPD21DsdupBems+roKm5tfit1XBxwHfJj2yD+d+gzFbxq/3IJVyohNZfFO48z4bomNAlQfze2ohJJDSClJO2TT3AKDsZv7Fitw7ErPx/6GYPZ4Jid7OdKoXjXh+tUdYPyvLPqIJ4uTkwZWDNLodQUKpHUAHmbNlG4ezeBjzyCzrXmlnSoagaTgZ8O/kTvhr1p6lM7ZilfkbgVkLwTBrwALh5aR2N3diRm8FfcGR4Z0Bx/Txetw6nVVCKxc9JsJm3mpzg3aoTviDu0DseurE5aTXphumNW+TUZ4K/XIKgNdByndTR2R0rJjFXx1Pdx4/7eDvglo5qpRGLnctesoTgujqDJjyGcHbQIYTmklETFRRFaJ5ReDXppHU71+2ceZCaqUigVWLn3NHtOZDFtcCvcnB1wJF81U4nEjkmTibRPZ+HSojl1br5Z63Dsyp60PezP2M/4NuPRCQf7NS7KgY3vQGgktLxB62jsTonRzHt/xhNWz5s7Ote+Uij2yMH+AmuW7BW/UZKYSNCUxxF69a2qrKi4KLydvbm1+a2X37m22fopFKTD4DdUKZRyRO04xrGMAp4bGoZep96f6qASiZ2SJSWkz56NW3g43jcM1jocu3I6/zRrj61leMvheDg7WCdzzinYOhvajYSGnbWOxu7kFBn4dN1hercIoF8rNRy6utg0kQghhgghDgohEoQQz5Xz+MdCiN3W2yEhRFaZx+4RQhy23u4ps72LEGKv9Zyfilpa7yBr6VIMKSkEPTHVYUo6VNbig4uRSMaGOeBQ6I0zwGyEQS9rHYld+mLjEc4WGHh+qOOUQrEHNuulE0Logc+AwUAysFMIsUJKeeDcPlLKJ8vsPwXLEr4IIfyBV4GugAT+sR57FpgDPAjsAFYCQ4BVtnodWjAXFZE+5wvcO3fGMzJS63DsSpGxiCWHltA/pD8h3g52/Ts1DnbNhx6PgF+o1tHYnVPZhXy7+Si3RzSgXUNVQqg62bJF0h1IkFImWtcuWQTcdon9xwILrT/fCKyVUmZak8daYIgQoj5QR0q5XUopgR+ohQtsnV2wEGNqqmqNlOOPxD/IKs5iQrgDDvn96zVw8Ya+T2sdiV36aM0hpISnblClUKqbLccNNgROlLmfDPQob0chRBOgKbD+Esc2tN6Sy9lea5jy8sn4+ms8e/XCs3t3rcOxK1JK5sfNp5VfK7oGd9U6nOp1NAYOrcZw/QySz2RTVHRG64jsisFk5saGJkY0b0jemWPEqbfniri5uRESEoLzVU4xsJcB6GOAJVJKU1WdUAgxCZgE0Lhx46o6rc1l/vA9prNnCXpiqtah2J2/T/9NQlYCb/R6w7FaamazpRRKnRCS69+It7c3oaGhjvUeXMbR9Hz0/kZaB3vX2kWrbEVKSUZGBsnJyTRtenWTN235jqcAjcrcD7FuK88Y/v+y1qWOTbH+fNlzSim/klJ2lVJ2DQqqGaM3TFlZZM79Dq9Bg3Dv0EHrcOxOVFwUfq5+DGvmYMvI7l8GJ3fBwJcoKjEQEBCgkkgZeUUGcosM1PV2VUnkKgghCAgIoKio6KrPYct3fSfQUgjRVAjhgiVZrLhwJyFEGOAHbCuz+U/gBiGEnxDCD7gB+FNKeQrIEUJcZx2tdTfwqw1fQ7XK+HYu5vz8aluMpiY5kXuCjSc2MrLVSFz1DlRvzFgM696A4PbQ4U4AlUTKkFJyKrsIF72OAE8H+r2oYtf6O2WzRCKlNAKTsSSFOOAnKeV+IcQbQoiys8jGAIusnefnjs0E3sSSjHYCb1i3ATwKfAMkAEeoJSO2jGlpZM6fT51hw3Br3UrrcOzOwviF6IWe0a1Hax1K9dr5LWQds5RCsbNFu3r06EFERASNGzcmKCiIiIgIIiIiSEpKuqLzLFu2jPj4+Ct+/j59+rB5eyyFBhPBPm7ornDy4QcffMCCBQuu+Hmr06hRo0hMTLzkPkePHsXT05NPPvmk3MePHDlC9+7dadGiBePGjcNgMFR5nDbtI5FSrsQyRLfstlcuuP9aBcfOBeaWsz0WaFd1UdqH9C+/QpaUEDRlstah2J18Qz7LDy9ncJPBBHsGax1O9SnMguj3oNkAaDFI62gusmPHDgDmzZtHbGwss2fPvqrzLFu2DJ1OR1hY2BUfm55XTD1nPb7uV9ZJbDAY+OGHH9i1a9cVP2d1evjhh3n//feZM2dOhftMmzaNoUOHVvj49OnTeeaZZxg5ciQPPPAA8+bN48EHH6zSONUFRTtgOHmSrMWL8Rl+Oy6hoVqHY3d+TfiVPEMe48MdbM2RzR9bksng17WO5IqtWrWKnj170rlzZ0aPHk1+fj5g+VALDw+nQ4cOPPvss8TExLBy5UqefPLJK27NGExmDGZJPR83oqKiaN++Pe3ateOFF14o3efLL7+kVatW9OjRgwceeIAnnngCgLVr19K9e3f01tJD27dvp0OHDkRERPD0008TEREBWL7NR0ZG0qlTJ7p06VKaPP/66y8GDBjArbfeSrNmzXjppZf44Ycf6NatGx06dCh9HRMmTOCxxx6jR48eNG/enOjoaO655x7CwsKYOHFiaZyTJk2ia9eutG3bljfeeKN0e//+/Vm9ejUmU/njkJYsWUJYWFiFSdhkMhEdHc3w4ZbVQ++55x5++eWXSr/HlWUvo7YcWtrnnwMQ9OijGkdif8zSzIL4BbQPbE/HoI5ah1N9sk7A9jnQYTTUL/91v/7bfg6czKnSpw1vUIdXb2l7TedITU3lnXfeYd26dXh4ePD2228zc+ZMJk6cyMqVK9m/fz9CCLKysvD19WXYsGGMHDmS22+v/JQwo8mMwSTxdHEiO/0ML730ErGxsfj4+HD99dfz+++/07FjR9555x3+/fdfPD096d+/P92tQ+q3bNlCly5dSs9333338f3339O9e3eefvr/5+nUr1+ftWvX4ubmRnx8PPfcc09pMtmzZw9xcXH4+PgQGhrKo48+ys6dO/nwww+ZPXs2H3zwAQDZ2dns2LGDpUuXcsstt7Bt2zbCwsLo3Lkz+/bto127drzzzjv4+/tjNBoZMGAAI0eOJDw8HL1eT2hoKPv27aNjx/N/D3Jzc/nwww9Zt24dM2bMKPd9SktLIzAwsDRhhoSEkJJS0Zinq6daJBorSUoie/kv+I4Zg3ODBlqHY3c2p2zmWM4xx1sBcYP1g2Hgi9rGcRW2bt3KgQMH6NWrFxEREURFRZGUlIS/vz86nY4HH3yQ5cuX43kNS0an5RUjkQR4ubBjxw4GDhxIYGAgzs7OjBs3jujo6NLtfn5+uLi4MHLkyNLjT506xbnRnOnp6ZSUlJQmmXHj/n99l+LiYiZOnEi7du0YM2YMBw6UFuagR48eBAcH4+bmRrNmzbjxxhsBaN++/Xktq1tuuaV0e4MGDQgPD0en0xEeHl6638KFC+ncuTOdO3cmLi7uvOepW7cuJ0+evOg9ePnll5k+fToeHtrXm1MtEo2lzZqNcHEh8KFJWodilxbELSDIPYgbmjhQufTTe2HPQug1BXwrngN1rS0HW5FSMmTIEH788ceLHouNjWXt2rX8/PPPzJkzhzVr1lR4nrIf7nfccQevvGLpXi0xmkjPK8FJp7vqtUbc3d0rNdz1ww8/pFGjRsyfPx+DwYCXl1fpY65lVivV6XSl93U6HUaj8aL9yu5Tdr/Dhw8zc+ZM/v77b3x9fZkwYcJ5sRUVFeHu7s6SJUt46623AEu/1N9//80vv/zCtGnTyMrKKj3/I488UnpsUFAQ6enpmEwm9Ho9ycnJNGxY9XO4VYtEQ0UHD5GzciX+EybgFBiodTh2JzE7kS0nt3Bn6ztx1jvQol5rXwU3H4icpnUkV6VXr15s2rSpdLRRfn4+hw8fJjc3l5ycHG6++WY+/vjj0o5ub29vcnNzLzqPi4sLu3fvZvfu3aVJBOBMTjECcLHOGenRowcbNmwgIyMDo9HIokWL6NevH927d2fDhg1kZWVhMBhYtmxZ6TnatGlDQkICQGlLJjY2FoBFixaV7pednU39+vURQvD9999TZnBplcnJycHb25s6depw6tQp/vzzz/MeP3z4MG3btmXkyJGl70dERARbt24lKSmJpKQkJk+ezCuvvHJeEgHQ6/VERkayfPlyAL7//ntuu+1SlaqujkokGkr79FN0np4ETLxf61Ds0oK4BTjrnBnVapTWoVSfI+vhyDroOx3c/bSO5qoEBwfz7bffMnr0aDp27EivXr04dOgQ2dnZ3HTTTXTs2JF+/frx0UcfATB27FhmzJhRqc72whIjZwtKCPByKV2KJSQkhDfffJP+/fsTERHBddddx0033UTjxo2ZPn063bp1o0+fPjRr1gwfH0sxx2HDhrFp06bS886dO5f77ruPTp06UVRUVLrf5MmT+eabb+jYsSNHjx49r0VRVTp37kx4eDhhYWHcfffd9O7du/SxkydP4uPjw5VOqr7xxhtJTU0F4P333+fdd9+lRYsW5OXlce+991Zl+BZSylp/69Kli7Q3BXv2yAOtw2TqZ59pHYpdyg9ShNMAACAASURBVCrKkt3md5MvxryodSjVx2SSck5vKT9uJ6WhqNxdDhw4UM1B2Zcjqblyf0qWNJpMldo/NzdXSillSUmJHDp0qFyxYkXpY7fccos8cuTIeftJKeVbb70lp02bVoVRX7333ntPzps3r1qeq7zfLSBWVuIzVrVINJL2yUz0fn74333P5Xd2QMsPL6fQWOhYVX73/mzpHxn4CjipWdplSSlJyy0mr9hIXW839LrKfXS9/PLLdOrUiQ4dOtC6dWtuLrNk9bvvvlvaib1ixQoiIiJo164d27Zt4/nnn7fJ67hSAQEBTJhg/38DqrNdA/l//03+1q3UfeYZ9F5XP3KltjKajSyMX0iX4C6E+V/5JLUayVAE69+0DPVtN0LraOxKXpGBk9lFFBlMeLk64e/lUuljP/744wofa9OmTenP48aNO2+0lr24//6acdlbJZJqJqUk7ZOZONWti984B1zhrxI2ntjIyfyT/F975x0eVZk+7PtJbxBISEF6pIoYmtIFOxpAfiwSV3+7trXtxy7qgrCuq358CysgKi5KUVCkKoiIggULRRfR4AaBhB56C+k9k5n3++OcZIeQkDI15L2vK1cmc9455znvTOY5bzn3O+n6SZ4OxX38tBByTsDdb0Itr7avdErKrJzNKSanyEKArw/tIkNoGuSvXWNeiE4kbqZg2zaKfvmF2BdfwCcoyNPheCXLU5dzVehVDGszzNOhuIfCTNj2CnS8DeKGejoaj2O1KdLziknPL0WA2KZBtAgLrLNLS+M+dCJxI+WtEf9WrWj2G919URX7MveRdC6JZ/o8g59PI/l4bpsNxbkNUoXiTJRSZBdaOJtbjMVqo3lIALFNg/D30y00b6eR/Kd6B3lfbaI4JYWW//wnElD7ft7GxPLU5QT7BTOm0xhPh+Ieso4Z3Vo974cY77zB0B0UlJRxJqeYwtIyQgL8aBcRQkig/npqKOhU7yaU1Ur6G28QEBdH+KiRng7HK8kszmTjkY2MjBtJeGC4p8NxD9/+A8QHbnqu5rJehjM08pYyGwuXrOTr7b9gsdpo0zyEq6NCa5VEBg8eTHJycr3jb+ga+e3btxMfH0/Pnj2Jj4/nk0+qXprJHRp5nUjcRO5nn1F6+DBRf/4T4utd60p4C6v3r6bUVtp4vFpp22D3h9D/jxDufG2Fq9mxYwfJyclMnTqVxMTEiruu29fCYG2zKc7nFrP/XB7rP1lH5qmjdI5pQvPQALcMppdr5BMTvXt9m3KNfFXEx8ezc+dOkpOT+fzzz3nsscew2WyXlCvXyB86dIiQkBDee+89p8epE4kbUBYL6XPfJLBbN5rc3oicUXXAYrXwwf4PGHjVQOKaxXk6HNdyMglW3AtLRkCTljD4KU9H5HSq08hPnDiRrt260e3aHkyeMpnU/+zg35s3MfX5yfTp3avOi2KVs2zZskankQ8JCcHPz2i5FRUVAVyicNEa+SuI7I/WYjlxgtbz5yF6ameVfHXsK9KL0nlp4EueDsU1KAVHt8HWVyBti6E/uelvcMOjhlerPnw+xbiB0ZnE9oA7X3ZoF9Vp5O/73YN88tkG1mz6N8EBfoSoYlrHRvFJPTTy9pw8ebJRauTBMC0/+uijHDt2jBUrVlQkxnK0Rv4KwVZSwoV58wju2ZOwoXpqZ3UsT11Ou6btGNxqsKdDcS5KwYEvYdHtsGQkpO+D2/8BT+2Boc82WJ/W5ahKI793/yEyy/wR8WHW88+w54dNxEQ2c8rxGqtGHgxB5t69e9mxYwfTpk2jtLTUgZqsP7pF4mKyVq6k7Nw5rpoxQ99IVQ2/pv/K7gu7mXLDFHzkCrm2sdkgdb0xtffsrxDeBu56BXr9DvyddP+Qgy0HV6FMjfyS998nI7+U83nF2GwQGRbAf3Ym8d2337B69Wrmz59fL428M7gSNPLl3W8A3bt3JzAwkJSUlIuevyI08iIyXET2i8ghEZlSTZlxIpIiIntFZIX53E0ikmz3Uywio81t74lImt22nlXt1xuwFRSQsfBtQgb0J7R/P0+H47UsS11GmH8YozvWr2vDq7BaIHklvNUPVj8AlkK4+y3483+MbixnJREvZuDAgWzevIVvf9rNmZwisJQguWdo4ltGYUG+wxr5yjRWjXxaWlrF2ElaWhoHDx6kXbt2F73WXRp5l7VIRMQXeBO4DTgJ/Cwi65VSKXZlOgF/BQYppbJEJBpAKfUd0NMsEwEcAuwvXSYppda4KnZnkbl0KdbMTKKfuvIGU53FuYJzbDq6iXu73kuofwP2jpWVQPJyY5317OMQcy2MXQzXjAafxjNLr9hipdA3jOdnzOHpxx8EWxk+IkyfPp1mTcMYM2YMJSUl2Gy2izTyjz/+OLNnz2bdunW1mvVlj71GXinFyJEjSUhIAKjQyEdERNClS5eLNPL2g93lGnk/Pz+GDBlykUZ+7NixLF68mISEBJdr5Nu1a1drjfyWLVuYNWsW/v7++Pr6smDBApo3N7pK77jjDpYuXUp0dDSzZs3it7/9LVOmTKFv374NSyMPDAC+tPv7r8BfK5WZCfyhhv08Biy3+/s9YGxdYvGERr4sO1vt63u9Ov7Ek24/dkNizs45qsd7PdTx3OOeDqV+lOQr9e+5Sr3SRakXmyq18Gal9m1UymZzyeG8VSNvsVrVqaxC9euJbLXnZLY6n1usrC6qg7qgNfK1xxGNvCvHSFoBJ+z+PglU7t/pDCAiPwC+wEtKqS8qlbkXeLXSc9NE5AXgG2CKUqrEaVE7iYzF72LLyyNqwp89HYrXUmItYc2BNQxtM5Q2Tdp4Opy6UZQNP78NP86DwgxoPwT+Zz50GAqNaCxMKUVmQSnncksos9mICA0gpmkQ/r7eMdb197//nc2bN1NcXMzw4cOr1MjHxcWxfv16Zs6cSVlZGe3bt3fJvRb1QWvka3/8TsAwoDWwVUR6KKWyAUSkJdADsO80/CtwFggAFgKTgalUQkQew2jN0LZt9eteu4KyjAwyly6l6V13EtS1kWjQ68HGIxvJKsnif7t5/z9KBQUX4Me34Ke3oSQXOt0OQyZC28Y3BpZfUsbp7CKKLVZCA/3oEB5CcICnv1IuRmvk3YMr3/VTgP1lZmvzOXtOAjuUUhYgTUQOYCSWn83t44CPze0AKKXOmA9LRORdYCJVoJRaiJFo6Nu3r/NHyC5DxsKFqOJiWoz/kzsP26BQSrEsdRkdm3XkhtgbPB1OzeSegX//C3a+C5YiuGYUDPmLsX5II6O0zMoZO71724gQwoO13r0x48pE8jPQSUQ6YCSQe4HKKX8d8FvgXRFpgdHVZS+W+S1GC6QCEWmplDojxqd2NLDHRfHXC8vZs2StXEX46NEExnXwdDheS9K5JA5kHeDFAS969xdQ1lH4/nVjIN1mhevGweCnIaqLpyNzO4bevYT0/BIEiGkaRJTWu2twYSJRSpWJyHiMbilfYLFSaq+ITMUYwFlvbrtdRFIAK8ZsrAwAEWmP0aLZUmnXy0UkChAgGXjCVedQHy68NQ+lFC3++EdPh+LVLE9dTnhgOAlxCZ4OpWrS98O2V43lb318odf/wqAJ0Ly9pyNzO0opsossnM0x9O7NTL17gNa7a0xc2qGplNoIbKz03At2jxXwjPlT+bVHMQbsKz9/s9MDdRKlx4+TvXYtzceNI6B1w5PwuYtT+af47sR3PNT9IYL9gj0dzsWc2WXcRJiyHvyDod8TMHA8NL3K05F5hMLSMk5nG3r34ABf2kaEEar17ppK6EsKJ5I+dy7i50fkE497OhSvZmXqSgTh3q73ejqU/3J8Byy/BxbcCIe/M8Y/ntoNw6c3yiRisdo4kVnIofP5lFpttG4eQseoi5OIMzTyAGvXrmXfvn11jrGxa+S/+OILevfuTY8ePejTpw+bN2+uslxGRga33HILnTp14o477iAnJ8fpcepLCydRfOAAuZ9+RuQjD+MfHe3pcLyWQkshaw+u5dZ2txIbGuvZYJQyBIpbXzGEisERcPPzcP2jEOwcD1RDw2ZTXMgv4XxeCQqIahJIdJMgfKsYBymXF7733nskJSUxd+7ceh1z7dq1+Pj40NWNMxzLNfLld9d7K+Ua+Xnz5l2yLTo6mg0bNtCyZUt27drFiBEjOHHixCXlpk2bxp133snEiRP5xz/+wcyZM5k2bZpT49QtEidx4V//wic0lAi7u2U1l7L+8HryLHmenfKrFOz/HN65Fd6/GzIOwR3T4ek9cOOkRplElFLkFFk4cD6Ps7nFhAX60TkmjJbhwVUmkZqoTiM/adIkrrnmGq677jomT57Mtm3b2LhxI08//XS9WjPlNEaNfO/evWnZsiVgCCHz8/OrXLTqk08+4YEHHgC0Rt6rKdq9h7xNX9Ni/Hj8ml95NldnYVM2lqcup3tkd+KjPDBt1maFlHXGIPq5PdCsLYx4zVjm1s/56gtXM+OnGezLrHuXUGVsSlFaZsNqU8SFd+Zv/afQJMi/3vurTiP/yCOPsHHjRvbu3YuIkJ2dTbNmzbhLa+TrrZEv58MPP6Rfv374+1/6vmVkZFQoVlq1asWZM2cuKeMoOpE4gfQ5c/Bt1oyIBx/wdChezfbT2zmae5Tpg6e7d8qv1QK/fgjfv2q0Plp0htHzocdY8K3/F2ZDRykotdoos9pAIMDPh2Yh/g4lEbhYIw+GxXfw4MFERETg4+PDo48+SkJCwkV3mTuCvUYeqNDIFxcXV2jkAcaOHcvx48cBQyPfq1cvoGqN/Ndffw0YGvnx48eza9cu/Pz8OHz4cMVxyzXywCUa+e3bt1eUq0ojD1Ro5K+99lpWrlzJokWLKCsr4/Tp06SkpFSUK9fIV5dIdu/ezfPPP8+mTZscrcp6oxOJgxQmJVHw/fdET5qIr51iWnMpy1KXERkUyR3t73DPAS3F8J+l8MMbkHPcWLTpniXQbeQVIVKcfMPker3OVqE1+a/ePbpJIH5O0pooUyO/dOnSS7YlJSWxadMmVq9ezbx587RG3kGN/PHjxxkzZgzLli2jQ4eq71uLjIwkPT2dqKgoTp06VdEd5kz0GIkDKKU4//rr+Ea1oLkX6hW8ibScNL4/9T2JXRIJ8A1w7cFK8o3kMec62DgRmsTCfavh8W3QvXHZeCuTV2zh4Ll8TmcXEezvS6eYMK5qFuy0JAKGRn7Lli0Vs40KCgo4ePAgeXl55Obmao18Jeqrkc/KyiIhIYFXXnmF/v37V7v/UaNGsWTJEsB1GnmdSByg4PsfKEraSYsnnsAn2Mvuh/AyVqSuwN/Hn3u63OO6gxRlweYZ8Pq1sOnvENUVHvgUHvkKOt/eqGSKlSmxWDl6oYC0CwWAon1kKB1ahBLk7/ykGhMTw6JFi0hMTCQ+Pp6BAwdy4MABcnJySEhIID4+nqFDh16kkZ8+fXq9B9vtNfI9e/akf//+JCQk0LZt2wqN/ODBg4mLi7tII79ly3/vdS7XyPfq1Yvi4uKLNPLvvPMO8fHxpKWluVwj//vf/77WGvk5c+aQlpbGiy++WDH1OiMjAzDGfMqnRj/33HNs2LCBTp06sXXrViZNmuT0cxBXZFhvo2/fvqr8asNZKKU4es84rJmZXP3F50iAi6+yGzC5pbncuvpWbmt3G9MGO3faIQD56fDjm/DTO1CaB53vNO4DaXO984/lYVJTUy+SDdaE1WbjfF4JF/JL8QGimwYSGRaITyNJqvn5+YSFhWGxWLj77rt58sknK8YsRo0axeuvv05cXFxFOTCmy2ZmZjJ79mxPhg7ArFmziI6Orph15Uqq+myJyE6lVN+aXqvHSOpJ3tdfU7xnDy2nTdNJpAY+PvgxRWVF3N/tfufuOOcU/PsN2LkEyoqNbqshfzHGQho5SimyCks5m2Pq3UMCiAn3Hr27u9AaefegWyT1QFmtpI0ejSqzEvfpesRP5+PqsNqsJHycQExIDEvuXOKcnWYeMUWKKwAF1yUaIsUWnZyzfy+mNi2SAlPvXmSxEhrgR8tmQYR4md5d433oFombyd24kZKDh2j16mydRGpgy8ktnMo/xTN9LtGp1Z3zqcY9IHvWgI8/9HnAECk2c+96M95KaZmNszlFZBdZ8Nd6d40b0d+CdURZLKT/ay6BXbvSZPhwT4fj9SxPXU5saCw3t3XAtXn6P4bGZN9n4B8KA/4PDBhvzMbSGHr3/BIu5BkLhWq9u8bd6ERSR7I//hjL8eO0fustxKdx9TfXlf2Z+/np7E881fsp/Hzq8VE7th22vQKHvoagcLjxWej/JIREOD/YBki51uRMud49OIDYcK1317gfnUjqgK2khAtvzSMo/jrCbhrm6XC8nhX7VhDkG8TYzmNr/yKl4PC3hsr92A8Q0gJueRGu/wMENXVdsA2MwtIyzmQXU1BaRrC/1rtrPIu+dKkD2R98QNnZs0Q/9ZTud66BrOIsNhzZwIirRxAeGF7zC2w22LcB3r4Zlo2BzDQY/rKhch/yjE4iJlabqtC7l5SZevdozyQRrZF3PZfTyJ8/f55hw4YRGhpaIaOsCndo5HUiqSW2wkIuLFhISL9+hA4Y4OlwvJ41B9ZQYi3h/q41TPm1WWH3Gpg/CFbdB0WZMHIOTEg2urECQtwTsJdTUmZl/pbDnMstJrvIQlSTQLrEhhERGuCxi5odO3aQnJzM1KlTSUxMrLjrun379nXaT30TiSOUa+QTExPdety6Uq6Rr4pyIeaMGTMuu49yjfzBgwcZMmQIM2fOdHqcOpHUksyly7BmZBD11ARPh+L1WGwWVu1fRf+W/enYvGPVhcpK4Zf3YW5f+OgRUDYY8zaM3wl9HmyQNl5XoJTiq71nuf21rbz8+T4C/XzoHF2ud/fef1+tkTfOw5Ua+bCwMAYNGkRQUNBl60Zr5L0Ea24uGYsWETZ0KCGmMVRTPd8c+4bzhed5oX8VfiRLkZFAfngDck9Cy3gYtxS6jgAv/mL0BAfO5TH10xS+P3SBjtFhvP/wDURaLxBoak3OTp9OSapzr+QDu3Ul1u6LuD5ojbz7NfKXQ2vkvYSMd9/FlpurWyO1ZFnqMto0acOQ1kP++2RJHvy8CLbPhYJ0aDvA6MLqeEujdmBVRXZhKa9tOsCyHccJDfDlpZHXcH//dvj7+pCaesHT4dWI1si7VyPvDbg0kYjIcGAO4Au8o5R6uYoy44CXAAXsUkrdZz5vBXabxY4rpUaZz3cAVgGRwE7gd0qpUledQ1lmJllL3qfJ8OEE1cFx1FjZc2EPu9J3Mfn6yfiIDxRmwo4FsGM+FGdD3E3GKoTtB9W8s0ZGmdXGip+O8+qmA+QWWbi/Xzuevq0zEaFVK3gcbTm4Cq2Rd59Gvja4QyPvskQiIr7Am8BtwEngZxFZr5RKsSvTCfgrMEgplSUi9oudFymlqqqpGcBrSqlVIjIfeAS4dEFjJ5Gx8G1sxcVE/flPrjrEFcWy1GWE+ocyOnYgfPV3SFoMpfnQJQFu/Au06lPzThoh3x+8wNTP9nLgXD4Dr47khZHX0DW2Yc5UGzhwIBMmTODIkSPExcVRUFDA6dOniY2Npbi4mBEjRjBw4EC6dOkC1KyRr4l+/foxceJEMjIyCA8PZ9WqVUycOJEePXrw7LPPkp2dTWhoKGvXrqVvX8P2UZ1Gvm/fvpdo5Dt27Oh2jfxwu5udyzXyUVFRjB1bh6n0JuUa+YkTJ7pMI+/KFskNwCGl1BEAEVkF3A2k2JV5FHhTKZUFoJQ6f7kdijE95WagfPGPJRitGZckEsu5c2StWEH4qFEExsW54hBXFOmF6XyZ9iWJQa0Ie2sAWEuh+xhj+m5Md0+H55UcvVDAtI2pbEo5R9uIEBb8rg+3XxPToKeX22vkS0uNzoLp06cTHBzMmDFjKCkpwWazXaSRf/zxx5k9ezbr1q2r86wve428UoqRI0eSkJAAUKGRj4iIoEuXLhdp5O0Hu8s18n5+fgwZMuQijfzYsWNZvHgxCQkJLtfIt2vXrtYa+fJzLywsxGKxsGbNGr755hu6dOnCQw89xIQJE+jZsyfPPfcc48aNY8GCBXTo0IEPPvjA6efgMmmjiIwFhiul/mD+/Tugn1JqvF2ZdcABYBBG99dLSqkvzG1lQDJQBryslFonIi2AH5VSHc0ybYDPlVLXVnH8x4DHANq2bdvn2LFjdT6H1Ym96bq7iBkP+5AZ3nD/sd1FkQ9k+MLHJ89zQG7kg4DfcMr3Kk+H5dUcyyjEz1cYf3NHHh7Uocb1QeqqkW/saI187WnI0kY/oBMwDGgNbBWRHkqpbKCdUuqUiMQB34rIbqDWd9IopRYCC8Gw/9YnOEtEU5L6WAloGkTslS9JdhwrDCiNZHHrmWT5xxCC8eZqqmdQxxY8OexqYppefgqnpn5ojbx7cGUiOQW0sfu7tfmcPSeBHUopC5AmIgcwvnt+VkqdAlBKHRGRzUAv4COgmYj4KaXKqtmn07hv3mYAHnLVATQajUt57bXXqt1mf/V93333cZ8XLpf98MMPezqEWuHKifs/A51EpIOIBAD3AusrlVmH0RrB7LbqDBwRkeYiEmj3/CAgRRn9cN8B5SNODwCfuPAcNBqNRlMDLkskZothPPAlkAp8qJTaKyJTRWSUWexLIENEUjASxCSlVAbQDUgSkV3m8y/bzfaaDDwjIocwpgAvctU5aDTeSGNYjE7jXhz9TOkVEjWaBkRaWhpNmjQhMjKyQc/s0ngPSikyMjLIy8ujQ4cOF21rKIPtGo2mDrRu3ZqTJ0+Snp7u6VA0VxBBQUG0bt263q/XiUSjaUD4+/tfctWo0XgabcnTaDQajUPoRKLRaDQah9CJRKPRaDQO0ShmbYlIOlB3R4pBC8Ab3d06rrqh46obOq66caXG1U4pVbXoy45GkUgcQUSSajP9zd3ouOqGjqtu6LjqRmOPS3dtaTQajcYhdCLRaDQajUPoRFIzCz0dQDXouOqGjqtu6LjqRqOOS4+RaDQajcYhdItEo9FoNA6hE0klRGSWiOwTkV9F5GMRaVZNueEisl9EDonIFDfEdY+I7BURm4hUOwtDRI6KyG4RSRYRl5sq6xCXu+srQkQ2ichB83fzaspZzbpKFpHKyxw4M57Lnr+IBIrIB+b2HSLS3lWx1DGuB0Uk3a6O/uCmuBaLyHkR2VPNdhGRN8y4fxWR3l4Q0zARybGrqxdcHZN53DYi8p2IpJj/ixOqKOPa+lJK6R+7H+B2wM98PAOYUUUZX+AwEAcEALuAa1wcVzegC7AZ6HuZckeBFm6srxrj8lB9zQSmmI+nVPU+mtvy3VBHNZ4/8Edgvvn4XuADL4nrQWCuuz5Pdse9EegN7Klm+13A54AA/TEWyPN0TMOAzzxQVy2B3ubjJhjLl1d+H11aX7pFUgml1FfKWEsF4EeMVRgrcwNwSCl1RClVCqwC7nZxXKlKqf2uPEZ9qGVcbq8vc/9LzMdLgNEuPt7lqM3528e7BrhFXO+J98T7UiuUUluBzMsUuRt4Xxn8iLFyaksPx+QRlFJnlFK/mI/zMNZ/alWpmEvrSyeSy/MwRhavTCvghN3fJ7n0jfMUCvhKRHaKyGOeDsbEE/UVo5Q6Yz4+C8RUUy5IRJJE5EcRcVWyqc35V5QxL2RyMBZucyW1fV9+Y3aHrBGRNlVs9wTe+j84QER2icjnItLd3Qc3u0R7ATsqbXJpfTVKjbyIfA3EVrHpb0qpT8wyfwPKgOXeFFctGKyUOiUi0cAmEdlnXkl5Oi6nc7m47P9QSikRqW56YjuzvuKAb0Vkt1LqsLNjbcB8CqxUSpWIyOMYraabPRyTt/ILxucpX0TuwlhKvJO7Di4iYcBHwFNKqVx3HRcaaSJRSt16ue0i8iAwArhFmR2MlTgF2F+ZtTafc2lctdzHKfP3eRH5GKP7wqFE4oS43F5fInJORFoqpc6YTfjz1eyjvL6OiMhmjKs5ZyeS2px/eZmTIuIHhAMZTo6jznEpY+nrct7BGHvyBlzymXIE+y9vpdRGEXlLRFoopVzu4BIRf4wkslwptbaKIi6tL921VQkRGQ48C4xSShVWU+xnoJOIdBCRAIzBUZfN+KktIhIqIk3KH2NMHKhyhomb8UR9rQceMB8/AFzSchKR5iISaD5uAQwCUlwQS23O3z7escC31VzEuDWuSv3oozD6372B9cDvzdlI/YEcu65MjyAiseXjWiJyA8b3q6svBjCPuQhIVUq9Wk0x19aXu2cYePsPcAijLzHZ/CmfSXMVsNGu3F0YsyMOY3TxuDqu/8Ho1ywBzgFfVo4LY/bNLvNnr7fE5aH6igS+AQ4CXwMR5vN9gXfMxwOB3WZ97QYecWE8l5w/MBXjggUgCFhtfv5+AuJcXUe1jOuf5mdpF/Ad0NVNca0EzgAW8/P1CPAE8IS5XYA3zbh3c5mZjG6MabxdXf0IDHRTXQ3GGBv91e576y531pe+s12j0Wg0DqG7tjQajUbjEDqRaDQajcYhdCLRaDQajUPoRKLRaDQah9CJRKPRaDQOoROJRuMERCTfwdevMe+uR0TCRGSBiBw2VTebRaSfiASIyFbzhkWNxmvQiUSj8TCmk8lXKXXEfOodDDlgJ6VUH+AhDKNzKca9MYmeiVSjqRqdSDQaJ2LeOTxLRPaIsS5Movm8j6nM2CfG+igbRWSs+bL7Me+8F5GrgX7A80opG4BSKk0ptcEsu84sr9F4DbqJrNE4lzFATyAeaAH8LCJbMfQr7YFrgGgM1chi8zWDMO6aBugOJCulrNXsfw9wvUsi12jqiW6RaDTOZTCGLdeqlDoHbMH44h8MrFZK2ZRSZzF0I+W0BNJrs3MzwZSWO9U0Gm9AJxKNxvMUYbi2wHA1xYuI72XKBwLFLo9Ko6klOpFoNM5lG5AoIr4iEoWxPOtPwA8YC0T5iEgMxrKs5aQCHQGUsRZKEvB/7Uyy7UUkwXwcCVxQSlncdUIaTU3oA9bpIAAAAMpJREFURKLROJePMSysu4BvgWfNrqyPMIyxKcAyjEWQcszXbODixPIHjBUdD4nIHuA9/rueyk1meY3Ga9D2X43GTYhImDJWz4vEaKUMUkqdFZFgjDGTQZcZZC/fx1pgilLqgBtC1mhqhZ61pdG4j89EpBkQAPw/s6WCUqpIRF7EWEP7eHUvNhefWqeTiMbb0C0SjUaj0TiEHiPRaDQajUPoRKLRaDQah9CJRKPRaDQOoROJRqPRaBxCJxKNRqPROIROJBqNRqNxiP8PQhU62UIZ/FUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "accuracy_s1 =np.array(accuracy_s).reshape(len(C_s),len(gamma_s))\n",
    "x_axis = np.log10(C_s)\n",
    "for j, gamma in enumerate(gamma_s):\n",
    "    pyplot.plot(x_axis, np.array(accuracy_s1[:,j]), label = ' Test - log(gamma)' + str(np.log10(gamma)))\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'accuracy' )\n",
    "pyplot.savefig('RBF_SVM_Otto.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 最优参数\n",
    "根据以上分析，最优参数是C=1,gamma = 0.01。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
